diff --git a/.github/workflows/action-smoke.yml b/.github/workflows/action-smoke.yml new file mode 100644 index 0000000..0a44000 --- /dev/null +++ b/.github/workflows/action-smoke.yml @@ -0,0 +1,76 @@ +name: Gate Action Smoke Test + +on: + push: + branches: [main, dev, "develop/august"] + paths: + - action.yml + - pyproject.toml + - src/framevitals/cli.py + - src/framevitals/operations.py + - .github/workflows/action-smoke.yml + pull_request: + paths: + - action.yml + - pyproject.toml + - src/framevitals/cli.py + - src/framevitals/operations.py + - .github/workflows/action-smoke.yml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + smoke: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Create stable fixture datasets + shell: bash + run: | + cat > reference.csv <<'CSV' + age,income,segment + 21,30000,basic + 34,62000,pro + 48,81000,pro + 52,90000,enterprise + CSV + cp reference.csv current.csv + + - name: Run local FrameVitals gate action + id: framevitals + uses: ./ + with: + current: current.csv + reference: reference.csv + output: gate-result.json + + - name: Verify action outputs + shell: bash + env: + STATUS: ${{ steps.framevitals.outputs.status }} + PASSED: ${{ steps.framevitals.outputs.passed }} + RESULT_PATH: ${{ steps.framevitals.outputs.result-path }} + run: | + test "$STATUS" = "pass" + test "$PASSED" = "true" + test "$RESULT_PATH" = "gate-result.json" + test -s gate-result.json + python - <<'PY' + import json + from pathlib import Path + + payload = json.loads(Path("gate-result.json").read_text(encoding="utf-8")) + assert payload["status"] == "pass" + assert payload["passed"] is True + assert payload["checks_run"] == ["drift"] + PY diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..481b32c --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,152 @@ +name: Performance Benchmark + +on: + push: + branches: [main, "develop/august"] + paths: + - "src/framevitals/**" + - "benchmarks/**" + - ".github/workflows/benchmark.yml" + workflow_dispatch: + inputs: + rows: + description: Rows in the synthetic profile benchmark workload + required: false + default: "50000" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + profile-scale: + name: profile scale benchmark + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install benchmark runtime + run: | + python -m pip install --upgrade pip + pip install -e ".[arrow]" + + - name: Run reproducible profile benchmark + env: + BENCHMARK_ROWS: ${{ inputs.rows || '50000' }} + run: | + python benchmarks/benchmark_profile_scale.py \ + --rows "$BENCHMARK_ROWS" \ + --numeric-columns 40 \ + --categorical-columns 5 \ + --scenarios numpy auto parquet \ + --output benchmark-results.json + + - name: Publish benchmark summary + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + payload = json.loads(Path("benchmark-results.json").read_text(encoding="utf-8")) + lines = [ + "## FrameVitals profile benchmark", + "", + "| Scenario | Seconds | Peak RSS (MB) | Backend |", + "| --- | ---: | ---: | --- |", + ] + for measurement in payload["measurements"]: + backend = measurement.get("backend_status", {}).get("selected", "unknown") + lines.append( + f"| {measurement['scenario']} | {measurement['elapsed_seconds']:.3f} " + f"| {measurement['peak_rss_mb']:.1f} | {backend} |" + ) + lines.extend([ + "", + "Results are measurements, not absolute pass/fail thresholds. Compare like-for-like runs.", + ]) + Path(os.environ["GITHUB_STEP_SUMMARY"]).write_text( + "\n".join(lines) + "\n", + encoding="utf-8", + ) + PY + + - name: Upload benchmark result + uses: actions/upload-artifact@v5 + with: + name: framevitals-profile-benchmark + path: benchmark-results.json + retention-days: 30 + + deep-8k-180: + name: 8k x 180 Deep benchmark + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install benchmark runtime + run: | + python -m pip install --upgrade pip + pip install -e "." + + - name: Run reproducible 8k x 180 Deep benchmark + env: + FRAMEVITALS_BACKEND: numpy + run: | + python benchmarks/benchmark_deep_pipeline.py \ + --rows 8000 \ + --columns 180 \ + --output deep-benchmark-results.json + + - name: Publish Deep benchmark summary + run: | + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + from pathlib import Path + + payload = json.loads(Path("deep-benchmark-results.json").read_text()) + print("## FrameVitals 8k x 180 Deep benchmark") + print() + print(f"- Wall time: **{payload['elapsed_seconds']:.3f}s**") + print(f"- Historical original: {payload['historical_original_seconds']:.3f}s") + print(f"- Speedup vs historical original: **{payload['speedup_vs_historical_original']:.2f}x**") + print(f"- 10x target: {payload['ten_x_target_seconds']:.3f}s") + print(f"- Target met: **{payload['ten_x_target_met']}**") + print() + print("### Pipeline stages") + for name, milliseconds in sorted(payload["pipeline_timings_ms"].items()): + if isinstance(milliseconds, (int, float)): + print(f"- {name}: {milliseconds / 1000:.3f}s") + PY + + - name: Upload Deep benchmark result + uses: actions/upload-artifact@v6 + with: + name: framevitals-deep-8k-180-benchmark + path: deep-benchmark-results.json + retention-days: 30 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..26001ff --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,45 @@ +name: CodeQL + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + schedule: + - cron: "17 4 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + analyze: + name: ${{ matrix.language }} security analysis + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + packages: read + security-events: write + strategy: + fail-fast: false + matrix: + language: + - python + - javascript-typescript + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + + - name: Analyze + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/developer-guardrails.yml b/.github/workflows/developer-guardrails.yml new file mode 100644 index 0000000..7b4eecd --- /dev/null +++ b/.github/workflows/developer-guardrails.yml @@ -0,0 +1,57 @@ +name: Developer Guardrails + +on: + pull_request: + paths: + - ".pre-commit-config.yaml" + - "pyproject.toml" + - "src/framevitals/**" + - "tests/**" + - ".github/workflows/developer-guardrails.yml" + push: + branches: [main, dev, "develop/august"] + paths: + - ".pre-commit-config.yaml" + - "pyproject.toml" + - "src/framevitals/**" + - "tests/**" + - ".github/workflows/developer-guardrails.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + hooks: + name: pre-commit + pre-push contracts + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install development environment + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Verify dependency consistency + run: python -m pip check + + - name: Run commit-stage hooks + run: pre-commit run --all-files --hook-stage pre-commit --show-diff-on-failure + + - name: Run pre-push public contract hooks + run: pre-commit run --all-files --hook-stage pre-push --show-diff-on-failure diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..46bf9ae --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,52 @@ +name: Docs + +on: + push: + branches: [main, dev, "develop/august"] + paths: + - "docs/**" + - "mkdocs.yml" + - "pyproject.toml" + - ".github/workflows/docs.yml" + pull_request: + paths: + - "docs/**" + - "mkdocs.yml" + - "pyproject.toml" + - ".github/workflows/docs.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: strict MkDocs build + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install documentation dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[docs]" + + - name: Verify dependency consistency + run: python -m pip check + + - name: Build documentation strictly + run: mkdocs build --strict diff --git a/.github/workflows/extensive-scale-benchmark.yml b/.github/workflows/extensive-scale-benchmark.yml new file mode 100644 index 0000000..7a31e39 --- /dev/null +++ b/.github/workflows/extensive-scale-benchmark.yml @@ -0,0 +1,273 @@ +name: Extensive Physical Scale Benchmark + +on: + push: + branches: ["develop/august"] + paths: + - ".github/workflows/extensive-scale-benchmark.yml" + - "benchmarks/benchmark_real_wide_parquet.py" + - "src/framevitals/**" + - "rust/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + stress: + name: ${{ matrix.label }} — native vs fallback full pipeline + runs-on: ubuntu-latest + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - label: tall-250k-x-10k + rows: 250000 + columns: 10000 + row_group_rows: 10000 + - label: wide-100k-x-25k + rows: 100000 + columns: 25000 + row_group_rows: 10000 + - label: tall-500k-x-10k + rows: 500000 + columns: 10000 + row_group_rows: 10000 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Configure Rust + run: | + rustup default stable + rustc --version + cargo --version + + - name: Show runner capacity + run: | + nproc + free -h + df -h . + + - name: Install benchmark and native build runtime + run: | + python -m pip install --upgrade pip + pip install -e ".[arrow]" + pip install "maturin>=1.14,<2" + + - name: Build and install FrameVitals native engine + run: | + maturin build \ + --release \ + --manifest-path rust/framevitals-py/Cargo.toml \ + --interpreter python \ + --out dist-native + pip install --force-reinstall --no-deps dist-native/*.whl + FRAMEVITALS_BACKEND=rust python - <<'PY' + from framevitals.backends import backend_status + + status = backend_status() + assert status["native_available"] is True, status + assert status["selected"] == "rust", status + print(status) + PY + + - name: Generate physical Parquet workload + run: | + python benchmarks/benchmark_real_wide_parquet.py \ + --generate-only \ + --dataset "${{ matrix.label }}.parquet" \ + --rows ${{ matrix.rows }} \ + --columns ${{ matrix.columns }} \ + --row-group-rows ${{ matrix.row_group_rows }} \ + --output "generation-${{ matrix.label }}.json" + ls -lh "${{ matrix.label }}.parquet" + df -h . + + - name: Run native Rust full analysis in all modes + env: + FRAMEVITALS_BACKEND: rust + run: | + for mode in quick standard deep research; do + echo "=== rust / ${{ matrix.label }} / ${mode} ===" + python benchmarks/benchmark_real_wide_parquet.py \ + --dataset "${{ matrix.label }}.parquet" \ + --mode "${mode}" \ + --output "${{ matrix.label }}-rust-${mode}.json" + done + + - name: Run NumPy fallback full analysis in all modes + env: + FRAMEVITALS_BACKEND: numpy + run: | + for mode in quick standard deep research; do + echo "=== numpy / ${{ matrix.label }} / ${mode} ===" + python benchmarks/benchmark_real_wide_parquet.py \ + --dataset "${{ matrix.label }}.parquet" \ + --mode "${mode}" \ + --output "${{ matrix.label }}-numpy-${mode}.json" + done + + - name: Validate routing, safety, accuracy, and backend parity + run: | + python - <<'PY' + import json + from pathlib import Path + + label = "${{ matrix.label }}" + failures = [] + payloads = {} + + for backend in ("rust", "numpy"): + for mode in ("quick", "standard", "deep", "research"): + path = Path(f"{label}-{backend}-{mode}.json") + payload = json.loads(path.read_text()) + payloads[(backend, mode)] = payload + + observed = payload["backend"]["numeric_backend"] + if observed != backend: + failures.append(f"{backend}/{mode}: routed to {observed!r}") + + safety = payload["safety"] + if safety["full_materialization"] is not False: + failures.append(f"{backend}/{mode}: full materialization occurred") + if safety["source_rows"] != ${{ matrix.rows }}: + failures.append(f"{backend}/{mode}: wrong source row count") + if safety["source_columns"] != ${{ matrix.columns }}: + failures.append(f"{backend}/{mode}: wrong source column count") + + accuracy = payload["accuracy"] + checked = accuracy["profiled_columns_checked"] + if accuracy["missing_count_exact_matches"] != checked: + failures.append(f"{backend}/{mode}: missing-count parity failed") + if accuracy["numeric_count_exact_matches"] != checked: + failures.append(f"{backend}/{mode}: numeric-count parity failed") + if accuracy["numeric_minmax_exact_matches"] != checked: + failures.append(f"{backend}/{mode}: min/max parity failed") + if (accuracy["profile_mean_max_abs_error"] or 0.0) > 0.01: + failures.append( + f"{backend}/{mode}: profile mean max error " + f"{accuracy['profile_mean_max_abs_error']}" + ) + if accuracy["health_missing_abs_error"] > 0.02: + failures.append( + f"{backend}/{mode}: health missingness error " + f"{accuracy['health_missing_abs_error']}" + ) + + deep_checked = accuracy["deep_numeric_columns_checked"] + if deep_checked: + if accuracy["deep_exact_once_columns"] != deep_checked: + failures.append(f"{backend}/{mode}: exact-once reuse failed") + if (accuracy["deep_mean_max_abs_error"] or 0.0) > 0.01: + failures.append( + f"{backend}/{mode}: deep exact mean error " + f"{accuracy['deep_mean_max_abs_error']}" + ) + + # The two backends should agree on the deterministic full-stream facts. + for mode in ("quick", "standard", "deep", "research"): + r = payloads[("rust", mode)]["accuracy"] + n = payloads[("numpy", mode)]["accuracy"] + for key in ( + "profiled_columns_checked", + "missing_count_exact_matches", + "numeric_count_exact_matches", + "numeric_minmax_exact_matches", + "deep_exact_once_columns", + ): + if r[key] != n[key]: + failures.append( + f"{mode}: backend parity mismatch for {key}: {r[key]} != {n[key]}" + ) + if abs((r["profile_mean_max_abs_error"] or 0.0) - (n["profile_mean_max_abs_error"] or 0.0)) > 0.01: + failures.append(f"{mode}: backend profile-mean accuracy diverged") + + if failures: + raise SystemExit("\n".join(failures)) + print("Extensive native/fallback full-pipeline checks passed.") + PY + + - name: Publish stress summary + if: always() + run: | + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + from pathlib import Path + + label = "${{ matrix.label }}" + gen_path = Path(f"generation-{label}.json") + if gen_path.exists(): + g = json.loads(gen_path.read_text()) + print(f"## {label}") + print() + print(f"- Physical shape: **{g['rows']:,} × {g['columns']:,}**") + print(f"- Logical cells: **{g['cells']:,}**") + print(f"- Dense int16 equivalent: **{g['dense_int16_raw_gb']:.2f} GB**") + print(f"- Parquet file size: **{g['file_size_mb']:.2f} MB**") + print(f"- Generation time: **{g['generation_seconds']:.3f}s**") + print() + + print("| Backend | Mode | Wall time | Profiled cols | Sample rows | Profile max mean err | Deep max mean err |") + print("|---|---|---:|---:|---:|---:|---:|") + for backend in ("rust", "numpy"): + for mode in ("quick", "standard", "deep", "research"): + path = Path(f"{label}-{backend}-{mode}.json") + if not path.exists(): + print(f"| {backend} | {mode} | failed | - | - | - | - |") + continue + p = json.loads(path.read_text()) + s = p["safety"] + a = p["accuracy"] + print( + f"| {backend} | {mode} | {p['analysis_seconds']:.3f}s | " + f"{s['profiled_columns']} | {s['working_sample_rows']} | " + f"{a['profile_mean_max_abs_error']} | {a['deep_mean_max_abs_error']} |" + ) + print() + + for backend in ("rust", "numpy"): + for mode in ("quick", "standard", "deep", "research"): + path = Path(f"{label}-{backend}-{mode}.json") + if not path.exists(): + continue + p = json.loads(path.read_text()) + print(f"### {backend} / {mode} stage timings") + for name, ms in sorted(p.get("pipeline_timings_ms", {}).items()): + if isinstance(ms, (int, float)): + print(f"- {name}: {ms / 1000:.3f}s") + print() + PY + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v6 + with: + name: framevitals-extensive-${{ matrix.label }} + path: | + generation-${{ matrix.label }}.json + ${{ matrix.label }}-rust-quick.json + ${{ matrix.label }}-rust-standard.json + ${{ matrix.label }}-rust-deep.json + ${{ matrix.label }}-rust-research.json + ${{ matrix.label }}-numpy-quick.json + ${{ matrix.label }}-numpy-standard.json + ${{ matrix.label }}-numpy-deep.json + ${{ matrix.label }}-numpy-research.json + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/extreme-streaming-benchmark.yml b/.github/workflows/extreme-streaming-benchmark.yml new file mode 100644 index 0000000..de67d78 --- /dev/null +++ b/.github/workflows/extreme-streaming-benchmark.yml @@ -0,0 +1,108 @@ +name: Extreme Streaming Benchmark + +on: + push: + branches: ["develop/august"] + paths: + - "src/framevitals/execution.py" + - "src/framevitals/planning_api.py" + - "src/framevitals/streaming_pipeline.py" + - "src/framevitals/health_score.py" + - "benchmarks/benchmark_extreme_streaming.py" + - ".github/workflows/extreme-streaming-benchmark.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + mode-comparison-100k-x-100k: + name: ${{ matrix.mode }} 100k x 100k virtual stream + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + mode: [quick, standard, deep, research] + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install streaming benchmark runtime + run: | + python -m pip install --upgrade pip + pip install -e ".[arrow]" + + - name: Run 100k x 100k benchmark + env: + FRAMEVITALS_BACKEND: numpy + run: | + python benchmarks/benchmark_extreme_streaming.py \ + --rows 100000 \ + --columns 100000 \ + --mode "${{ matrix.mode }}" \ + --output "extreme-${{ matrix.mode }}-results.json" + + - name: Publish benchmark summary + if: always() + env: + MODE: ${{ matrix.mode }} + run: | + FILE="extreme-${MODE}-results.json" + if [[ ! -f "$FILE" ]]; then + echo "## ${MODE} 100k x 100k benchmark" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + echo "Benchmark did not produce a result file." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + import os + from pathlib import Path + + mode = os.environ["MODE"] + payload = json.loads(Path(f"extreme-{mode}-results.json").read_text()) + workload = payload["workload"] + safety = payload["safety"] + print(f"## FrameVitals {mode.title()} 100k x 100k benchmark") + print() + print(f"- Logical cells: **{workload['cells']:,}**") + print(f"- Dense float64 equivalent: **{workload['dense_float64_raw_gb']:.1f} GB raw**") + print(f"- Plan: **{payload['plan_seconds']:.3f}s**") + print(f"- Analysis: **{payload['analysis_seconds']:.3f}s**") + print(f"- Scale class: **{payload['scale_class']}**") + print(f"- Profiled columns: **{safety['profiled_columns']} / {safety['source_columns']}**") + print(f"- Working sample rows: **{safety['working_sample_rows']:,}**") + print(f"- All 100,000 source rows scanned: **{safety['all_source_rows_scanned']}**") + print(f"- Full materialization: **{safety['full_materialization']}**") + print(f"- Unbounded 100k-column request: **{safety['unbounded_width_requested']}**") + print() + print("### Pipeline stages") + for name, milliseconds in sorted(payload.get("pipeline_timings_ms", {}).items()): + if isinstance(milliseconds, (int, float)): + print(f"- {name}: {milliseconds / 1000:.3f}s") + PY + + - name: Upload benchmark result + if: always() + uses: actions/upload-artifact@v6 + with: + name: framevitals-100k-x-100k-${{ matrix.mode }} + path: extreme-${{ matrix.mode }}-results.json + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml new file mode 100644 index 0000000..d3a2462 --- /dev/null +++ b/.github/workflows/interop.yml @@ -0,0 +1,62 @@ +name: Interoperability + +on: + push: + branches: [main, dev, "develop/august"] + paths: + - "src/framevitals/sources.py" + - "src/framevitals/duckdb_source.py" + - "src/framevitals/analysis_api.py" + - "src/framevitals/streaming_*.py" + - "tests/test_arrow_memory.py" + - "tests/test_duckdb_source.py" + - "pyproject.toml" + - ".github/workflows/interop.yml" + pull_request: + paths: + - "src/framevitals/sources.py" + - "src/framevitals/duckdb_source.py" + - "src/framevitals/analysis_api.py" + - "src/framevitals/streaming_*.py" + - "tests/test_arrow_memory.py" + - "tests/test_duckdb_source.py" + - "pyproject.toml" + - ".github/workflows/interop.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + arrow-duckdb: + name: Arrow + DuckDB (Python 3.12) + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install interoperability capabilities + run: | + python -m pip install --upgrade pip + pip install -e ".[duckdb,dev]" + + - name: Verify dependency consistency + run: python -m pip check + + - name: Run interoperability tests + env: + FRAMEVITALS_BACKEND: numpy + run: pytest -v tests/test_arrow_memory.py tests/test_duckdb_source.py diff --git a/.github/workflows/minimum-dependencies.yml b/.github/workflows/minimum-dependencies.yml new file mode 100644 index 0000000..723a871 --- /dev/null +++ b/.github/workflows/minimum-dependencies.yml @@ -0,0 +1,86 @@ +name: Minimum Dependencies + +on: + pull_request: + paths: + - "pyproject.toml" + - "src/framevitals/**" + - "tests/**" + - ".github/workflows/minimum-dependencies.yml" + push: + branches: [main, dev, "develop/august"] + paths: + - "pyproject.toml" + - "src/framevitals/**" + - "tests/**" + - ".github/workflows/minimum-dependencies.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lower-bounds: + name: Python 3.11 lower bounds + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install declared minimum core dependencies + run: | + python -m pip install --upgrade pip + pip install \ + "numpy==1.26.0" \ + "pandas==2.2.0" \ + "scipy==1.13.0" \ + "statsmodels==0.14.0" \ + "scikit-learn==1.5.0" \ + "pytest>=8.2" \ + "hypothesis>=6.100" + pip install --no-deps -e . + + - name: Verify dependency consistency + run: python -m pip check + + - name: Verify installed lower bounds + run: | + python - <<'PY' + import numpy + import pandas + import scipy + import sklearn + import statsmodels + + expected = { + "numpy": "1.26.0", + "pandas": "2.2.0", + "scipy": "1.13.0", + "sklearn": "1.5.0", + "statsmodels": "0.14.0", + } + observed = { + "numpy": numpy.__version__, + "pandas": pandas.__version__, + "scipy": scipy.__version__, + "sklearn": sklearn.__version__, + "statsmodels": statsmodels.__version__, + } + assert observed == expected, (observed, expected) + PY + + - name: Run core test suite at lower bounds + run: pytest -q diff --git a/.github/workflows/mixed-ground-truth.yml b/.github/workflows/mixed-ground-truth.yml new file mode 100644 index 0000000..6c9f5b9 --- /dev/null +++ b/.github/workflows/mixed-ground-truth.yml @@ -0,0 +1,143 @@ +name: Mixed Ground Truth Validation + +on: + push: + branches: ["develop/august"] + paths: + - ".github/workflows/mixed-ground-truth.yml" + - "tests/test_mixed_ground_truth_pipeline.py" + - "tests/test_release_hardening.py" + - "tests/test_parquet_streaming.py" + - "tests/test_target_intelligence.py" + - "tests/test_target_intelligence_v2.py" + - "tests/test_semantic_types.py" + - "tests/test_relationship_graph.py" + - "tests/test_fast_anomaly.py" + - "tests/test_deep_statistics.py" + - "src/framevitals/**" + - "rust/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + name: native/fallback mixed statistical ground truth + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Configure Rust + run: | + rustup default stable + rustc --version + cargo --version + + - name: Install test and Arrow dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev,arrow]" + pip install "maturin>=1.14,<2" + + - name: Build and install FrameVitals native engine + run: | + maturin build \ + --release \ + --manifest-path rust/framevitals-py/Cargo.toml \ + --interpreter python \ + --out dist-native + pip install --force-reinstall --no-deps dist-native/*.whl + FRAMEVITALS_BACKEND=rust python - <<'PY' + from framevitals.backends import backend_status + + status = backend_status() + assert status["native_available"] is True, status + assert status["selected"] == "rust", status + print(status) + PY + + - name: NumPy fallback mixed ground-truth suite + env: + FRAMEVITALS_BACKEND: numpy + run: | + pytest -q \ + tests/test_mixed_ground_truth_pipeline.py \ + tests/test_release_hardening.py \ + tests/test_parquet_streaming.py \ + tests/test_target_intelligence.py \ + tests/test_target_intelligence_v2.py \ + tests/test_semantic_types.py \ + tests/test_relationship_graph.py \ + tests/test_fast_anomaly.py \ + tests/test_deep_statistics.py \ + --junitxml=mixed-ground-truth-numpy.xml + + - name: Rust native mixed ground-truth suite + env: + FRAMEVITALS_BACKEND: rust + run: | + pytest -q \ + tests/test_mixed_ground_truth_pipeline.py \ + tests/test_release_hardening.py \ + tests/test_parquet_streaming.py \ + tests/test_target_intelligence.py \ + tests/test_target_intelligence_v2.py \ + tests/test_semantic_types.py \ + tests/test_relationship_graph.py \ + tests/test_fast_anomaly.py \ + tests/test_deep_statistics.py \ + --junitxml=mixed-ground-truth-rust.xml + + - name: Publish validation summary + if: always() + run: | + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + from pathlib import Path + import xml.etree.ElementTree as ET + + print("## Mixed ground-truth validation") + print() + print("Coverage: mixed Parquet streaming, exact missingness/moments, finite-value fallback semantics, sampling provenance, quantile tolerance, categorical/date detection, target intelligence, semantic types, relationship discovery, anomaly recall, and backend parity through the same test contracts.") + print() + print("| Backend | Tests | Failures | Errors | Skipped |") + print("|---|---:|---:|---:|---:|") + for backend in ("numpy", "rust"): + path = Path(f"mixed-ground-truth-{backend}.xml") + if not path.exists(): + print(f"| {backend} | unavailable | - | - | - |") + continue + root = ET.parse(path).getroot() + attrs = root.attrib + print( + f"| {backend} | {attrs.get('tests', '?')} | {attrs.get('failures', '?')} | " + f"{attrs.get('errors', '?')} | {attrs.get('skipped', '?')} |" + ) + PY + + - name: Upload validation reports + if: always() + uses: actions/upload-artifact@v6 + with: + name: framevitals-mixed-ground-truth + path: | + mixed-ground-truth-numpy.xml + mixed-ground-truth-rust.xml + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index b45eae6..140857d 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -2,9 +2,16 @@ name: Package on: push: - branches: [main, dev] + branches: [main, dev, "develop/august"] pull_request: +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest @@ -20,52 +27,108 @@ jobs: with: python-version: "3.12" + - name: Configure Rust + run: | + rustup default stable + rustc --version + cargo --version + - name: Install build tools run: | python -m pip install --upgrade pip - pip install build twine + pip install build twine "maturin==1.14.1" - - name: Build package - run: python -m build + - name: Build portable fallback distribution + run: python -m build --outdir dist-fallback - - name: Verify wheel contents + - name: Build native wheel + run: | + maturin build \ + --locked \ + --release \ + --manifest-path rust/framevitals-py/Cargo.toml \ + --interpreter python \ + --out dist-native \ + --compatibility pypi + + - name: Verify fallback and native wheel contents run: | python - <<'PY' from pathlib import Path import zipfile - wheels = list(Path("dist").glob("*.whl")) - assert len(wheels) == 1, f"Expected exactly one wheel, found {len(wheels)}" + fallback = list(Path("dist-fallback").glob("*.whl")) + native = list(Path("dist-native").glob("*.whl")) + assert len(fallback) == 1, f"Expected one fallback wheel, found {len(fallback)}" + assert len(native) == 1, f"Expected one native wheel, found {len(native)}" - wheel = wheels[0] - with zipfile.ZipFile(wheel) as archive: - names = archive.namelist() + def names(path: Path) -> list[str]: + with zipfile.ZipFile(path) as archive: + return archive.namelist() - assert any(name.startswith("framevitals/") for name in names), ( - "framevitals package missing from wheel" - ) - assert not any(name.startswith("modules/") for name in names), ( - "legacy modules package leaked into wheel" - ) - print(f"Verified distribution boundary: {wheel.name}") + fallback_names = names(fallback[0]) + native_names = names(native[0]) + + for wheel, wheel_names in ((fallback[0], fallback_names), (native[0], native_names)): + assert any(name.startswith("framevitals/") for name in wheel_names), ( + f"framevitals package missing from {wheel.name}" + ) + assert "framevitals/py.typed" in wheel_names, ( + f"PEP 561 marker missing from {wheel.name}" + ) + assert not any(name.startswith("modules/") for name in wheel_names), ( + f"legacy modules package leaked into {wheel.name}" + ) + + native_suffixes = (".so", ".pyd", ".dylib") + assert not any( + name.startswith("framevitals/_native") and name.endswith(native_suffixes) + for name in fallback_names + ), "Portable fallback unexpectedly contains a native extension" + assert any( + name.startswith("framevitals/_native") and name.endswith(native_suffixes) + for name in native_names + ), "Native wheel does not contain framevitals._native" + + print(f"Portable fallback: {fallback[0].name}") + print(f"Native wheel: {native[0].name}") PY - name: Validate distributions - run: python -m twine check dist/* + run: twine check dist-fallback/* dist-native/* - - name: Smoke-test built wheel + - name: Verify pip prefers the native wheel when compatible run: | + mkdir -p dist-install + cp dist-fallback/*.whl dist-install/ + cp dist-native/*.whl dist-install/ python -m venv /tmp/framevitals-wheel-smoke - /tmp/framevitals-wheel-smoke/bin/python -m pip install --no-deps dist/*.whl + /tmp/framevitals-wheel-smoke/bin/python -m pip install --upgrade pip + /tmp/framevitals-wheel-smoke/bin/python -m pip install numpy pandas + VERSION=$(python - <<'PY' + import tomllib + with open("pyproject.toml", "rb") as handle: + print(tomllib.load(handle)["project"]["version"]) + PY + ) cd /tmp + /tmp/framevitals-wheel-smoke/bin/python -m pip install \ + --no-index \ + --find-links "$GITHUB_WORKSPACE/dist-install" \ + --no-deps \ + "framevitals==$VERSION" /tmp/framevitals-wheel-smoke/bin/python - <<'PY' from importlib.metadata import version import framevitals + from framevitals.backends import backend_status installed = version("framevitals") assert installed == framevitals.__version__, ( f"metadata version {installed} != package version {framevitals.__version__}" ) - print(f"Imported FrameVitals {installed} from built wheel") + status = backend_status() + assert status["native_available"] is True, status + assert status["selected"] == "rust", status + print(f"Imported FrameVitals {installed} with native backend: {status}") PY /tmp/framevitals-wheel-smoke/bin/framevitals --version diff --git a/.github/workflows/performance-guardrail.yml b/.github/workflows/performance-guardrail.yml new file mode 100644 index 0000000..c06f4be --- /dev/null +++ b/.github/workflows/performance-guardrail.yml @@ -0,0 +1,103 @@ +name: Performance Guardrail + +on: + workflow_dispatch: + pull_request: + paths: + - "src/framevitals/profiler.py" + - "src/framevitals/streaming_profile.py" + - "src/framevitals/sources.py" + - "src/framevitals/focused.py" + - "src/framevitals/native_core.py" + - "tests/test_release_hardening.py" + - "rust/**" + - "benchmarks/performance_guardrail.py" + - "benchmarks/performance_budgets.json" + - ".github/workflows/performance-guardrail.yml" + push: + branches: [main, dev, "develop/august"] + paths: + - "src/framevitals/profiler.py" + - "src/framevitals/streaming_profile.py" + - "src/framevitals/sources.py" + - "src/framevitals/focused.py" + - "src/framevitals/native_core.py" + - "tests/test_release_hardening.py" + - "rust/**" + - "benchmarks/performance_guardrail.py" + - "benchmarks/performance_budgets.json" + - ".github/workflows/performance-guardrail.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + profile-guardrails: + name: profiling time + RSS ceilings + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install profiling capabilities + run: | + python -m pip install --upgrade pip + pip install -e ".[arrow]" + + - name: Verify dependency consistency + run: python -m pip check + + - name: Run performance guardrails + env: + FRAMEVITALS_BACKEND: numpy + run: | + python benchmarks/performance_guardrail.py \ + --output performance-guardrail.json + + - name: Publish job summary + if: always() + run: | + if [[ ! -f performance-guardrail.json ]]; then + exit 0 + fi + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + from pathlib import Path + + payload = json.loads(Path("performance-guardrail.json").read_text()) + print("## FrameVitals performance guardrail") + print() + print("| Scenario | Time (s) | Time budget | Peak RSS (MB) | RSS budget | Result |") + print("| --- | ---: | ---: | ---: | ---: | --- |") + for item in payload["measurements"]: + budget = item["budget"] + result = "PASS" if item["passed"] else "FAIL" + print( + f"| {item['name']} | {item['elapsed_seconds']:.3f} | " + f"{budget['max_seconds']:.1f} | {item['peak_rss_mb']:.1f} | " + f"{budget['max_peak_rss_mb']:.1f} | {result} |" + ) + PY + + - name: Upload guardrail measurements + if: always() + uses: actions/upload-artifact@v6 + with: + name: performance-guardrail + path: performance-guardrail.json + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml new file mode 100644 index 0000000..b750b24 --- /dev/null +++ b/.github/workflows/platform-smoke.yml @@ -0,0 +1,84 @@ +name: Platform Smoke + +on: + pull_request: + paths: + - "pyproject.toml" + - "src/framevitals/**" + - "tests/test_public_api.py" + - "tests/test_public_surface_contract.py" + - "tests/test_source_inspection.py" + - "tests/test_diagnostic_results.py" + - "tests/test_snapshots.py" + - "tests/test_cli.py" + - "tests/test_cli_monitoring.py" + - "tests/test_cli_system_info.py" + - ".github/workflows/platform-smoke.yml" + push: + branches: [main, dev, "develop/august"] + paths: + - "pyproject.toml" + - "src/framevitals/**" + - "tests/test_public_api.py" + - "tests/test_public_surface_contract.py" + - "tests/test_source_inspection.py" + - "tests/test_diagnostic_results.py" + - "tests/test_snapshots.py" + - "tests/test_cli.py" + - "tests/test_cli_monitoring.py" + - "tests/test_cli_system_info.py" + - ".github/workflows/platform-smoke.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + smoke: + name: ${{ matrix.os }} / Python 3.12 + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install core package and test tools + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + + - name: Verify dependency consistency + run: python -m pip check + + - name: Verify CLI install + run: | + framevitals --version + framevitals --help + + - name: Run cross-platform public smoke suite + run: >- + python -m pytest -q + tests/test_public_api.py + tests/test_public_surface_contract.py + tests/test_source_inspection.py + tests/test_diagnostic_results.py + tests/test_snapshots.py + tests/test_cli.py + tests/test_cli_monitoring.py + tests/test_cli_system_info.py diff --git a/.github/workflows/plugin-example.yml b/.github/workflows/plugin-example.yml new file mode 100644 index 0000000..ba735bd --- /dev/null +++ b/.github/workflows/plugin-example.yml @@ -0,0 +1,87 @@ +name: Check Plugin Example + +on: + push: + branches: [main, dev, "develop/august"] + paths: + - "src/framevitals/checks.py" + - "src/framevitals/plugins.py" + - "src/framevitals/operations.py" + - "examples/check_plugin/**" + - ".github/workflows/plugin-example.yml" + pull_request: + paths: + - "src/framevitals/checks.py" + - "src/framevitals/plugins.py" + - "src/framevitals/operations.py" + - "examples/check_plugin/**" + - ".github/workflows/plugin-example.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + smoke: + name: install and discover external checks + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: | + pyproject.toml + examples/check_plugin/pyproject.toml + + - name: Install FrameVitals and example provider + run: | + python -m pip install --upgrade pip + pip install -e . + pip install --no-deps -e examples/check_plugin + + - name: Verify dependency consistency + run: python -m pip check + + - name: Discover and run provider checks + run: | + python - <<'PY' + import pandas as pd + import framevitals as fv + + checks = fv.discover_checks() + assert [check.name for check in checks] == [ + "positive revenue", + "preferred plan domain", + ] + + passing = pd.DataFrame({ + "revenue": [10.0, 20.0, 30.0], + "plan": ["basic", "pro", "enterprise"], + }) + passing_result = fv.gate(passing, custom_checks=checks) + assert passing_result.status == "pass" + + failing = pd.DataFrame({ + "revenue": [10.0, -5.0, 30.0], + "plan": ["basic", "legacy", "enterprise"], + }) + failing_result = fv.gate(failing, custom_checks=checks) + assert failing_result.status == "fail" + assert failing_result["checks"]["custom"]["summary"] == { + "checks": 2, + "passed": 0, + "warnings": 1, + "errors": 1, + } + PY diff --git a/.github/workflows/polars-interop.yml b/.github/workflows/polars-interop.yml new file mode 100644 index 0000000..6e8aefa --- /dev/null +++ b/.github/workflows/polars-interop.yml @@ -0,0 +1,61 @@ +name: Polars Interoperability + +on: + push: + branches: [main, dev, "develop/august"] + paths: + - "src/framevitals/sources.py" + - "src/framevitals/analysis_api.py" + - "src/framevitals/focused.py" + - "src/framevitals/result.py" + - "tests/test_arrow_memory.py" + - "tests/test_polars_interop.py" + - ".github/workflows/polars-interop.yml" + pull_request: + paths: + - "src/framevitals/sources.py" + - "src/framevitals/analysis_api.py" + - "src/framevitals/focused.py" + - "src/framevitals/result.py" + - "tests/test_arrow_memory.py" + - "tests/test_polars_interop.py" + - ".github/workflows/polars-interop.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + polars-arrow: + name: Polars via Arrow C Stream (Python 3.12) + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install FrameVitals Arrow capability and Polars + run: | + python -m pip install --upgrade pip + pip install -e ".[arrow,dev]" + pip install "polars>=1,<2" + + - name: Verify dependency consistency + run: python -m pip check + + - name: Run Polars interoperability tests + env: + FRAMEVITALS_BACKEND: numpy + run: pytest -v tests/test_arrow_memory.py tests/test_polars_interop.py diff --git a/.github/workflows/real-wide-parquet-benchmark.yml b/.github/workflows/real-wide-parquet-benchmark.yml new file mode 100644 index 0000000..8e1da24 --- /dev/null +++ b/.github/workflows/real-wide-parquet-benchmark.yml @@ -0,0 +1,229 @@ +name: Real Wide Parquet Benchmark + +on: + push: + branches: ["develop/august"] + paths: + - "benchmarks/benchmark_real_wide_parquet.py" + - ".github/workflows/real-wide-parquet-benchmark.yml" + - "src/framevitals/**" + - "rust/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + name: Native vs fallback full pipeline 100k x 10k real Parquet + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Configure Rust + run: | + rustup default stable + rustc --version + cargo --version + + - name: Install benchmark and native build runtime + run: | + python -m pip install --upgrade pip + pip install -e ".[arrow]" + pip install "maturin>=1.14,<2" + + - name: Build and install FrameVitals native engine + run: | + maturin build \ + --release \ + --manifest-path rust/framevitals-py/Cargo.toml \ + --interpreter python \ + --out dist-native + pip install --force-reinstall --no-deps dist-native/*.whl + FRAMEVITALS_BACKEND=rust python - <<'PY' + from framevitals.backends import backend_status + + status = backend_status() + assert status["native_available"] is True, status + assert status["selected"] == "rust", status + print(status) + PY + + - name: Generate real 100k x 10k Parquet dataset + run: | + python benchmarks/benchmark_real_wide_parquet.py \ + --generate-only \ + --dataset wide-100k-x-10k.parquet \ + --rows 100000 \ + --columns 10000 \ + --row-group-rows 10000 \ + --output generation.json + + - name: Run native Rust full analysis in all modes + env: + FRAMEVITALS_BACKEND: rust + run: | + for mode in quick standard deep research; do + echo "=== rust / ${mode} ===" + python benchmarks/benchmark_real_wide_parquet.py \ + --dataset wide-100k-x-10k.parquet \ + --mode "${mode}" \ + --output "real-wide-rust-${mode}.json" + done + + - name: Run NumPy fallback full analysis in all modes + env: + FRAMEVITALS_BACKEND: numpy + run: | + for mode in quick standard deep research; do + echo "=== numpy / ${mode} ===" + python benchmarks/benchmark_real_wide_parquet.py \ + --dataset wide-100k-x-10k.parquet \ + --mode "${mode}" \ + --output "real-wide-numpy-${mode}.json" + done + + - name: Validate backend routing and full-pipeline accuracy + run: | + python - <<'PY' + import json + from pathlib import Path + + failures = [] + for backend in ("rust", "numpy"): + for mode in ("quick", "standard", "deep", "research"): + payload = json.loads( + Path(f"real-wide-{backend}-{mode}.json").read_text() + ) + observed_backend = payload["backend"]["numeric_backend"] + if observed_backend != backend: + failures.append( + f"{backend}/{mode}: routed to {observed_backend!r}" + ) + + accuracy = payload["accuracy"] + checked = accuracy["profiled_columns_checked"] + if accuracy["missing_count_exact_matches"] != checked: + failures.append(f"{backend}/{mode}: missing-count parity failed") + if accuracy["numeric_count_exact_matches"] != checked: + failures.append(f"{backend}/{mode}: numeric-count parity failed") + if accuracy["numeric_minmax_exact_matches"] != checked: + failures.append(f"{backend}/{mode}: min/max parity failed") + if (accuracy["profile_mean_max_abs_error"] or 0.0) > 0.01: + failures.append( + f"{backend}/{mode}: profile mean max error " + f"{accuracy['profile_mean_max_abs_error']}" + ) + if accuracy["health_missing_abs_error"] > 0.02: + failures.append( + f"{backend}/{mode}: health missingness error " + f"{accuracy['health_missing_abs_error']}" + ) + + deep_checked = accuracy["deep_numeric_columns_checked"] + if deep_checked: + if accuracy["deep_exact_once_columns"] != deep_checked: + failures.append( + f"{backend}/{mode}: exact-once reuse missing for " + f"{deep_checked - accuracy['deep_exact_once_columns']} columns" + ) + if (accuracy["deep_mean_max_abs_error"] or 0.0) > 0.01: + failures.append( + f"{backend}/{mode}: deep exact mean error " + f"{accuracy['deep_mean_max_abs_error']}" + ) + + if payload["safety"]["full_materialization"] is not False: + failures.append(f"{backend}/{mode}: full materialization occurred") + + if failures: + raise SystemExit("\n".join(failures)) + print("Native and fallback full-pipeline accuracy checks passed.") + PY + + - name: Publish benchmark summary + if: always() + run: | + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + from pathlib import Path + + generation_path = Path("generation.json") + if generation_path.exists(): + generation = json.loads(generation_path.read_text()) + print("## Real 100k x 10k Parquet dataset") + print() + print(f"- Logical cells: **{generation['cells']:,}**") + print(f"- Dense int16 equivalent: **{generation['dense_int16_raw_gb']:.2f} GB**") + print(f"- Parquet file size: **{generation['file_size_mb']:.2f} MB**") + print(f"- Generation time: **{generation['generation_seconds']:.3f}s**") + print(f"- Row groups: **{generation['row_groups']}**") + print() + + print("## Native Rust vs NumPy fallback") + print() + print("| Backend | Mode | Wall time | Profiled cols | Sample rows | Profile mean max error | Deep mean max error |") + print("|---|---|---:|---:|---:|---:|---:|") + for backend in ("rust", "numpy"): + for mode in ("quick", "standard", "deep", "research"): + path = Path(f"real-wide-{backend}-{mode}.json") + if not path.exists(): + print(f"| {backend} | {mode.title()} | failed | - | - | - | - |") + continue + payload = json.loads(path.read_text()) + safety = payload["safety"] + accuracy = payload["accuracy"] + print( + f"| {backend} | {mode.title()} | {payload['analysis_seconds']:.3f}s | " + f"{safety['profiled_columns']} | {safety['working_sample_rows']} | " + f"{accuracy['profile_mean_max_abs_error']} | " + f"{accuracy['deep_mean_max_abs_error']} |" + ) + print() + + for backend in ("rust", "numpy"): + for mode in ("quick", "standard", "deep", "research"): + path = Path(f"real-wide-{backend}-{mode}.json") + if not path.exists(): + continue + payload = json.loads(path.read_text()) + print(f"### {backend} / {mode.title()} pipeline stages") + for name, milliseconds in sorted(payload.get("pipeline_timings_ms", {}).items()): + if isinstance(milliseconds, (int, float)): + print(f"- {name}: {milliseconds / 1000:.3f}s") + print() + PY + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v6 + with: + name: framevitals-real-wide-100k-x-10k-native-vs-numpy + path: | + generation.json + real-wide-rust-quick.json + real-wide-rust-standard.json + real-wide-rust-deep.json + real-wide-rust-research.json + real-wide-numpy-quick.json + real-wide-numpy-standard.json + real-wide-numpy-deep.json + real-wide-numpy-research.json + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 588cdae..da73af8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,9 +4,12 @@ on: release: types: [published] +permissions: + contents: read + jobs: - build: - name: Build distribution + build-fallback: + name: Build portable fallback runs-on: ubuntu-latest steps: @@ -18,41 +21,176 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.x" + python-version: "3.12" - name: Install build tools run: | python -m pip install --upgrade pip python -m pip install build twine - - name: Build distributions - run: python -m build + - name: Verify release version across Python and Rust packages + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + PYTHONPATH=src python - <<'PY' + import os + import tomllib + from pathlib import Path + + import framevitals + + tag = os.environ["RELEASE_TAG"] + release_version = tag[1:] if tag.startswith("v") else tag + with Path("pyproject.toml").open("rb") as handle: + project_version = tomllib.load(handle)["project"]["version"] + with Path("rust/framevitals-core/Cargo.toml").open("rb") as handle: + core_version = tomllib.load(handle)["package"]["version"] + with Path("rust/framevitals-py/Cargo.toml").open("rb") as handle: + bridge_version = tomllib.load(handle)["package"]["version"] + package_version = framevitals.__version__ + assert Path("rust/Cargo.lock").is_file(), "Rust workspace lockfile is missing" - - name: Validate distributions + versions = { + "release tag": release_version, + "pyproject": project_version, + "python package": package_version, + "rust core": core_version, + "rust bridge": bridge_version, + } + assert len(set(versions.values())) == 1, f"Release versions disagree: {versions}" + print(f"Release version verified across Python/Rust packages: {project_version}") + PY + + - name: Build portable fallback wheel and sdist + run: python -m build --outdir dist + + - name: Validate fallback distributions run: python -m twine check dist/* - - name: Store distributions - uses: actions/upload-artifact@v5 + - name: Store fallback distributions + uses: actions/upload-artifact@v6 with: - name: python-package-distributions + name: python-package-fallback path: dist/ + if-no-files-found: error + + build-native: + name: Native wheel — ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64 + os: ubuntu-latest + target: x86_64-unknown-linux-gnu + manylinux: "2_28" + - name: macos-arm64 + os: macos-latest + target: aarch64-apple-darwin + manylinux: "off" + - name: macos-x86_64 + os: macos-15-intel + target: x86_64-apple-darwin + manylinux: "off" + - name: windows-x86_64 + os: windows-latest + target: x86_64-pc-windows-msvc + manylinux: "off" + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python 3.11 ABI floor + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Build native abi3 wheel + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + command: build + maturin-version: v1.14.1 + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + args: >- + --locked + --release + --manifest-path rust/framevitals-py/Cargo.toml + --interpreter python + --out dist + --compatibility pypi + + - name: Verify native wheel payload + shell: bash + run: | + python - <<'PY' + from pathlib import Path + import zipfile + + wheels = list(Path("dist").glob("*.whl")) + assert len(wheels) == 1, f"Expected one native wheel, found {len(wheels)}" + wheel = wheels[0] + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + suffixes = (".so", ".pyd", ".dylib") + assert "framevitals/py.typed" in names, f"PEP 561 marker missing from {wheel.name}" + assert any( + name.startswith("framevitals/_native") and name.endswith(suffixes) + for name in names + ), f"Native extension missing from {wheel.name}" + print(f"Verified native distribution: {wheel.name}") + PY + + - name: Store native wheel + uses: actions/upload-artifact@v6 + with: + name: python-package-native-${{ matrix.name }} + path: dist/*.whl + if-no-files-found: error publish: - name: Publish distribution to PyPI - needs: build + name: Publish distributions to PyPI + needs: [build-fallback, build-native] runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/p/framevitals permissions: id-token: write + contents: read steps: - - name: Download distributions + - name: Download all release distributions uses: actions/download-artifact@v6 with: - name: python-package-distributions + pattern: python-package-* path: dist/ + merge-multiple: true + + - name: Show release payload + run: ls -lh dist/ + + - name: Validate complete release payload + run: | + python -m pip install --upgrade twine + python -m twine check dist/* + python - <<'PY' + from pathlib import Path + + wheels = list(Path("dist").glob("*.whl")) + sdists = list(Path("dist").glob("*.tar.gz")) + native = [path for path in wheels if "none-any" not in path.name] + fallback = [path for path in wheels if "none-any" in path.name] + + assert len(sdists) == 1, f"Expected one sdist, found {len(sdists)}" + assert len(fallback) == 1, f"Expected one portable fallback wheel, found {len(fallback)}" + assert len(native) == 4, f"Expected four native wheels, found {len(native)}: {native}" + print("Release payload contains one fallback wheel, one sdist, and four native wheels.") + PY - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46dde21..3892b7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,9 +2,16 @@ name: Tests on: push: - branches: [main, dev] + branches: [main, dev, "develop/august"] pull_request: +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: quality: name: package quality @@ -34,8 +41,7 @@ jobs: - name: Check undefined-name and scope errors run: ruff check src/framevitals tests --select F821,F822,F823 - - name: Report broader dead-code lint debt - continue-on-error: true + - name: Check unused imports and assignments run: ruff check src/framevitals tests --select F401,F841 core: @@ -86,6 +92,152 @@ jobs: - name: Run tests run: pytest -v + native-core: + name: native core (Rust + Python bridge) + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Configure Rust tooling + run: | + rustup default stable + rustup component add rustfmt clippy + rustc --version + cargo --version + + - name: Check Rust formatting + run: cargo fmt --manifest-path rust/Cargo.toml --all -- --check + + - name: Test Rust workspace + run: cargo test --manifest-path rust/Cargo.toml --workspace --all-targets --all-features + + - name: Lint Rust workspace + run: cargo clippy --manifest-path rust/Cargo.toml --workspace --all-targets --all-features -- -D warnings + + - name: Install native build tooling + run: | + python -m pip install --upgrade pip + pip install \ + "maturin>=1.14,<2" \ + "numpy>=1.26" \ + "pandas>=2.2" \ + "pyarrow>=25.0,<26" + + - name: Build native Python wheel + run: | + maturin build \ + --release \ + --manifest-path rust/framevitals-py/Cargo.toml \ + --interpreter python \ + --out dist-native + + - name: Verify native Python bridge and streaming profiler routing + run: | + pip install --force-reinstall --no-deps dist-native/*.whl + FRAMEVITALS_BACKEND=rust python - <<'PY' + from pathlib import Path + from tempfile import TemporaryDirectory + + import numpy as np + import pandas as pd + import pyarrow as pa + import pyarrow.parquet as pq + + import framevitals + from framevitals import _native + from framevitals.profiler import build_profile + + values = np.ascontiguousarray( + np.array([1.0, 2.0, np.nan, np.inf, 4.0, 4.0], dtype=np.float64) + ) + state = _native.numeric_state_f64(values) + assert state["backend"] == "rust" + assert state["count"] == 4 + assert state["missing"] == 1 + assert state["infinite"] == 1 + assert abs(state["mean"] - 2.75) < 1e-12 + + native_profile = _native.numeric_profile_f64(values, stream_id=17) + assert native_profile["cardinality_estimate"] >= 3 + assert native_profile["quantiles"]["p50"] is not None + assert native_profile["reservoir"] + + accumulator = _native.NumericAccumulator(stream_id=29) + accumulator.update_f64(values[:3]) + accumulator.update_f64(values[3:]) + accumulated = accumulator.snapshot() + assert accumulator.observations == len(values) + assert accumulated["count"] == state["count"] + assert accumulated["missing"] == state["missing"] + assert accumulated["infinite"] == state["infinite"] + + public_profile = build_profile(pd.DataFrame({ + "value": [1.0, 2.0, None, 4.0, 4.0], + "other": [10.0, 20.0, 30.0, 40.0, 50.0], + })) + assert public_profile["numeric_summary_metadata"]["backend"] == "rust" + assert public_profile["numeric_summary_metadata"]["approximate_quantiles"] is True + assert public_profile["missing_counts"]["value"] == 1 + assert public_profile["numeric_summary"]["value"]["count"] == 4 + + with TemporaryDirectory() as directory: + path = Path(directory) / "native-stream.parquet" + frame = pd.DataFrame({ + "value": np.arange(20_000, dtype=np.float64), + "other": np.arange(20_000, dtype=np.float64) * 3.0, + "group": [f"g-{index % 5}" for index in range(20_000)], + }) + frame.loc[::97, "value"] = np.nan + pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=777) + streamed = framevitals.profile(path) + assert streamed["streaming_metadata"]["full_materialization"] is False + assert streamed["streaming_metadata"]["numeric_backend"] == "rust" + assert streamed["numeric_summary_metadata"]["backend"] == "rust" + assert streamed["missing_counts"]["value"] == int(frame["value"].isna().sum()) + assert streamed["numeric_summary"]["value"]["count"] == int(frame["value"].notna().sum()) + + print(_native.backend_info()) + PY + + arrow-streaming: + name: Arrow streaming (Python 3.12, NumPy fallback) + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install Arrow capability + run: | + python -m pip install --upgrade pip + pip install -e ".[arrow,dev]" + + - name: Verify dependency consistency + run: python -m pip check + + - name: Run streaming source tests without native extension + env: + FRAMEVITALS_BACKEND: numpy + run: pytest -v tests/test_parquet_streaming.py tests/test_statistics_streaming.py tests/test_streaming_drift.py tests/test_csv_streaming.py tests/test_arrow_memory.py tests/test_cli_gate_streaming.py + optional-features: name: optional features (Python 3.12) runs-on: ubuntu-latest @@ -116,9 +268,15 @@ jobs: python - <<'PY' import flask import lightgbm + import matplotlib import ollama + import openpyxl + import pydantic + import pyarrow import pyod + import seaborn import shap + import xlrd import xgboost print("Optional dependency smoke test passed") PY @@ -128,7 +286,9 @@ jobs: pytest -v \ tests/test_ai_agent.py \ tests/test_ml_engine.py \ - tests/test_model_diagnostics.py + tests/test_model_diagnostics.py \ + tests/test_visualization_engine.py \ + tests/test_web_dependency_boundaries.py frontend: name: frontend build diff --git a/.gitignore b/.gitignore index e3f92b4..9391314 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ frontend/vite.config.js frontend/vite.config.d.ts # Runtime artifacts +.framevitals/ uploads/* !uploads/.gitkeep cleaned/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4b87833 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +minimum_pre_commit_version: "4.0.0" + +default_install_hook_types: + - pre-commit + - pre-push + +repos: + - repo: local + hooks: + - id: framevitals-critical-ruff + name: FrameVitals critical Python lint + entry: python -m ruff check --select E9,F821,F822,F823 src/framevitals tests + language: system + pass_filenames: false + files: ^(src/framevitals|tests)/.*\.py$ + + - id: framevitals-compile + name: FrameVitals Python compile check + entry: python -m compileall -q src/framevitals + language: system + pass_filenames: false + files: ^src/framevitals/.*\.py$ + + - id: framevitals-public-contracts + name: FrameVitals public contract tests + entry: >- + python -m pytest -q + tests/test_public_surface_contract.py + tests/test_provenance.py + tests/test_source_inspection.py + tests/test_check_plugins.py + language: system + pass_filenames: false + always_run: true + stages: [pre-push] diff --git a/CHANGELOG.md b/CHANGELOG.md index 181e82a..d5fc975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,17 +2,111 @@ All notable user-facing changes to FrameVitals are documented here. -FrameVitals follows semantic versioning while the public API matures. The 0.x series may still include breaking changes, which will be called out in release notes. +FrameVitals follows semantic versioning while the public API matures. The 0.x series may still include breaking changes, which are called out in release notes. ## Unreleased -Development continues on the `dev` branch. Changes intended for the next release should be documented here before they are promoted to `main`. +No user-facing changes are currently queued beyond 0.2.0. -### Added +## 0.2.0 - 2026-08-17 + +FrameVitals 0.2.0 is the first release built around the source-aware Arrow/Rust execution architecture rather than treating every dataset as a pandas-first workload. + +### Highlights + +- Added bounded Arrow streaming for large Parquet, CSV/TSV, PyArrow, Arrow C Stream, Polars-through-Arrow, and optional DuckDB relation inputs. +- Added a native Rust execution backend with direct Arrow `RecordBatch` profiling, mergeable numeric state, native categorical sketches, and full-stream log-quantile sketches. +- Added exact-once reuse so downstream Deep/Research diagnostics consume already-known full-stream facts instead of recomputing weaker bounded-sample estimates. +- Added exact full-stream count, missingness, mean, variance/std, min/max, skewness, and excess kurtosis through mergeable central moments up to M4. +- Replaced fixed evenly spaced row sampling with deterministic stratified-jitter sampling to reduce periodic/structured-data aliasing while preserving reproducibility and row order. +- Added explicit execution provenance for full-stream, projected-column, sketch, bounded-sample, and materialized operations. +- Added physical large-scale benchmark coverage up to 500,000 × 10,000 (5 billion logical cells) with native/fallback routing, accuracy, exact-once, and no-full-materialization checks. +- Added mixed statistical ground-truth validation across native and fallback execution. +- PyPI publishing now builds native ABI3 wheels for common Linux/macOS/Windows targets while retaining a portable pure-Python fallback wheel and source distribution. + +### Analysis and public APIs + +- Added focused APIs for `profile`, `roles`, `health`, `ml_readiness`, `quality`, `statistics`, `anomalies`, `relationships`, and `target_analysis` so callers do not need to run unrelated pipeline stages. +- Added `framevitals.plan()` for previewing execution budgets, source capabilities, applicable work, and bounded planning behavior. +- Added `framevitals.inspect_source()` and the matching CLI command for source metadata/capability inspection without analysis. +- Added data-contract inference with `framevitals.infer_contract()` and structured validation with `framevitals.validate()`. +- Added `framevitals.gate()` plus a reusable GitHub Action for CI-friendly contract, drift, and custom-check verdicts. +- Added `framevitals.check()`, `run_checks()`, and opt-in third-party check discovery through Python entry points. +- Added compact versioned analysis snapshots, snapshot history, and snapshot-to-snapshot monitoring diffs. +- Added structured dict-compatible result objects including analysis, diagnostic, drift, validation, check, and gate results. +- Added `system-info`, snapshot, monitoring, planning, inspection, validation, and gate CLI workflows. + +### Execution architecture + +- Full `framevitals.analyze()` now resolves generic dataset sources and uses bounded source-aware execution when the source supports streaming/projection. +- Ultra-wide sources use deterministic schema projection under explicit cell/column budgets instead of scanning every source cell blindly. +- Streaming analysis performs one authoritative full-source profile scan and schedules only genuinely row-dependent modules on the bounded working sample. +- Numeric native execution accepts Arrow record batches directly through the Arrow C Data/PyCapsule interface, avoiding the former Arrow → NumPy float64 → per-column Python bridge on supported types. +- Streaming numeric profiling uses a specialized moments + log-quantile state instead of paying for unrelated HLL/heavy-hitter/reservoir work on every numeric cell. +- Parquet sources cache metadata/schema/file handles per source to avoid repeated metadata parsing. +- Research inference uses adaptive statistically defensible methods rather than blindly running expensive resampling on large bounded samples. +- Execution budgets now bound expensive quality, deep-statistics, anomaly, time-series, relationship, bootstrap, and distribution work by source scale and analysis mode. + +### Correctness and statistical fidelity + +- Full-stream missing counts, numeric counts, min/max, and moments are reused downstream with explicit provenance. +- Deep/Research shape summaries can reuse exact full-stream skewness/kurtosis while sample-dependent distribution fits, tests, intervals, and relationship diagnostics remain explicitly sample-scoped. +- Streaming ML-readiness, health, roles, quality, relationships, statistics, and anomaly results now distinguish exact/full-stream facts from bounded estimates. +- Categorical native profiling reports full-stream approximate cardinality/heavy-hitter provenance rather than pretending native sketches were row samples. +- Anderson-Darling normality diagnostics support modern and legacy SciPy contracts without relying on a fixed critical-value index. +- Large-source sampling regression tests cover periodic aliasing and native/fallback provenance contracts. + +### Performance and scale validation + +FrameVitals benchmarks are not advertised as universal speedups. Results depend on mode, backend, source shape, storage, and statistical fidelity. + +Validated physical workloads include 1B, 2.5B, and 5B logical cells. On the 5B 500k × 10k workload, the native full-stream profiling kernel remains competitive while retaining full-stream native quantile sketches where the fallback may choose bounded row-sample quantiles for cost. All published stress workflows enforce no full materialization and correctness tolerances. + +### Same-dataset release comparison vs 0.1.0 + +For an end-to-end release comparison, `v0.1.0` (`3da1432`) and the 0.2.0 release candidate (`05b11e5`) were run through the same public `framevitals.analyze(path, mode=..., artifacts=False)` API on one deterministic physical 10,000 × 64 CSV (640,000 cells). Both environments used Python 3.11 with the same pinned NumPy/pandas/SciPy/statsmodels/scikit-learn stack. FrameVitals 0.2.0 used its native Rust backend. Each version/mode received one warm-up followed by three measured runs in interleaved ABBAAB order; all 10,000 rows and all 64 columns were validated in every run. + +| Mode | 0.1.0 median wall | 0.2.0 median wall | Speedup | 0.1.0 median peak RSS | 0.2.0 median peak RSS | Memory change | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Quick | 1.644 s | 0.691 s | **2.38×** | 234.7 MB | 239.5 MB | 2.1% higher | +| Standard | 141.084 s | 0.839 s | **168.06×** | 2736.7 MB | 264.9 MB | **90.3% lower** | + +The measured Quick ranges were 1.620–1.648 s for 0.1.0 and 0.686–0.692 s for 0.2.0. Standard ranges were 140.709–141.130 s for 0.1.0 and 0.823–0.841 s for 0.2.0. The large Standard improvement is a release-level architectural result: 0.2.0 replaces 0.1.0's fully materialized, unbounded heavy-statistics path with bounded/adaptive execution and native streaming. It should not be interpreted as a same-algorithm microbenchmark. + +The exact same serialized CSV was then graded against independent pandas/SciPy ground truth. The full Standard pipeline still processed all 10,000 rows and all 64 columns in both releases; the accuracy comparison tracks five representative numeric columns (`c000`, `c001`, `c007`, `c031`, `c063`) plus Pearson correlation for `c000`↔`c001`. + +| Tracked metric | 0.1.0 max absolute error | 0.2.0 max absolute error | +| --- | ---: | ---: | +| Count / missing / min / max | **0** | **0** | +| Mean | 0.0005000 | 0.0005000 | +| Standard deviation | 0.0004745 | 0.0004745 | +| Skewness / excess kurtosis | 4.92×10⁻⁷ | 4.92×10⁻⁷ | +| Pearson correlation | 0.0001469 | 0.0001469 | +| q25 / median / q75 | **0** | 4.196 max; 1.589 mean | + +For every tested non-quantile metric, 0.2.0 had **exactly the same measured absolute error as 0.1.0 at the published output precision**. The quantile difference is intentional: 0.2.0 uses a full-stream native log-quantile sketch configured for 1% relative accuracy instead of materializing exact profile quartiles. Across the 15 tracked q25/median/q75 values, the maximum absolute error was 4.196 units and the mean was 1.589 units; normalized by each approximately 2,000-unit observed column range, that is **0.210% max** and **0.0795% mean**. Thus the measured 168.06× Standard speedup and 90.3% RSS reduction preserved the tested exact facts, moments, shape statistics, and correlation fidelity while trading exact profile quartiles for explicitly approximate streaming estimates. + +Timing/RSS evidence is committed as `benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json` from GitHub Actions run `32010158292`. Same-dataset accuracy evidence is committed as `benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json`; the legacy/current accuracy run was `32014979365`, with a current-only shape-field extractor correction verified in run `32015665811`. + +A separate 100,000 × 64 (6.4M-cell) stress run completed all Quick repetitions at a 5.097 s vs 1.447 s median (**3.52× faster**) and 406.1 MB vs 282.7 MB median peak RSS (**30.4% lower**) for 0.1.0 vs 0.2.0. The repeated Standard comparison intentionally has no formal speedup claim: the 30-minute workflow limit expired because completed 0.1.0 Standard passes took about 640–649 s and roughly 12.4 GB peak RSS each, while observed 0.2.0 Standard passes completed in about 1.58–2.06 s at roughly 303–308 MB. That incomplete stress evidence is GitHub Actions run `32008727705`. + +### Packaging and compatibility + +- Added Python 3.11, 3.12, and 3.13 core coverage plus Windows/macOS smoke testing. +- Added dedicated Arrow fallback, Rust/native, minimum-dependency, optional-feature, frontend, package-quality, documentation, and performance CI lanes. +- Added PEP 561 `py.typed` packaging support. +- Moved Excel, plotting, AI, Arrow, DuckDB, and other heavier capabilities behind explicit optional dependency groups where appropriate. +- Release CI verifies the GitHub tag, Python package version, `pyproject.toml`, Rust core version, and Rust bridge version before publication. +- Package CI verifies both fallback/native wheel contents and proves that pip prefers a compatible native wheel when both are available. + +### Fixed -- Data-contract inference through `framevitals.infer_contract()` -- Structured contract validation through `framevitals.validate()` -- `framevitals infer-contract` and `framevitals validate` CLI commands for CI-friendly data gates +- Fixed periodic sampling aliasing caused by fixed evenly spaced global-row samples. +- Fixed projected streaming missingness/quality semantics so denominators describe the profiled projection rather than silently using the true source width. +- Fixed duplicate core recomputation in the streaming pipeline. +- Fixed stale sampling/native-categorical provenance assertions and public execution labels. +- Fixed SciPy Anderson-Darling compatibility and a pandas string-dtype selection deprecation. +- Fixed several optional-dependency/import boundaries so core/lightweight usage does not eagerly require plotting, AI, web, or Excel stacks. ## 0.1.0 - 2026-08-15 @@ -20,40 +114,12 @@ First public alpha release of FrameVitals. ### Added -- Installable `framevitals` package under `src/framevitals/` -- Public `framevitals.analyze()` API for pandas DataFrames and supported dataset files -- Public `framevitals.compare()` API for reference-vs-current drift analysis -- `framevitals` command-line interface with analyze and compare commands -- Data-health and ML-readiness scoring -- Structural profiling and semantic column-role inference -- Missingness, duplicate, cardinality, statistical, anomaly, target-aware, time-series, and text diagnostics -- Drift comparison using PSI plus numeric/categorical statistical tests -- Optional artifact generation for reusable Python workflows -- Optional ML, AI, and Flask/React web dependency groups -- Python 3.11, 3.12, and 3.13 test matrix -- Package-build, wheel-boundary, Twine, and clean-install validation -- PyPI Trusted Publishing release workflow -- Dependabot configuration for Python, npm, and GitHub Actions -- Open-source contributor, security, conduct, issue, and pull-request guidance - -### Changed - -- Project identity and package documentation moved from DataLens AI to FrameVitals -- Reusable dashboard/report helpers moved into the canonical package -- Top-level package import and CLI version path load the analytics pipeline lazily -- Heavy ML, Ollama, and web dependencies moved into optional extras -- Reusable Python analysis no longer writes cleaned datasets or charts unless `artifacts=True` -- Repository layout is package-first, with the Flask API and React dashboard kept as optional interfaces -- Frontend module documentation now points directly at canonical `src/framevitals/` implementations -- Development setup standardized on `.venv` - -### Removed - -- Unused runtime dependencies including Optuna, imbalanced-learn, Pingouin, Plotly, Missingno, fpdf2, Jinja2, Joblib, and Loguru -- Academic report, presentation, and whitepaper artifacts -- Generated TypeScript compiler metadata and generated Vite JavaScript/config declarations -- Tracked runtime-output directories for uploads, reports, and cleaned datasets -- Large legacy demo CSVs and their one-off inspection harness -- Redundant Streamlit console and its configuration -- Legacy shell launcher/install scripts and duplicate `requirements.txt` development wrapper -- Deprecated top-level `modules/` compatibility namespace after application imports moved to `framevitals.*` +- Installable `framevitals` package under `src/framevitals/`. +- Public `framevitals.analyze()` API for pandas DataFrames and supported dataset files. +- Public `framevitals.compare()` API for reference-vs-current drift analysis. +- `framevitals` command-line interface with analyze and compare commands. +- Data-health and ML-readiness scoring. +- Structural profiling and semantic column-role inference. +- Missingness, duplicate, cardinality, statistical, anomaly, target-aware, time-series, and text diagnostics. +- Drift comparison using PSI plus numeric/categorical statistical tests. +- Optional artifact generation and ML/AI/web dependency groups. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 213dde2..4af8d37 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,8 @@ Thanks for considering a contribution to FrameVitals. +FrameVitals is being built as a package-first data-health and quality-gate library. Contributions are most valuable when they make the public workflow more reliable, source-aware, transparent, and maintainable rather than simply adding more unrelated diagnostics. + ## Development setup ```bash @@ -12,6 +14,8 @@ python -m venv .venv source .venv/bin/activate python -m pip install --upgrade pip pip install -e ".[all,dev]" +pre-commit install +pre-commit install --hook-type pre-push ``` On Windows PowerShell, activate the environment with: @@ -20,7 +24,14 @@ On Windows PowerShell, activate the environment with: .venv\Scripts\Activate.ps1 ``` -Run the test suite before making changes: +Run the same fast local guardrails that CI enforces: + +```bash +pre-commit run --all-files +pre-commit run --all-files --hook-stage pre-push +``` + +Run the full test suite before opening a substantial pull request: ```bash pytest @@ -36,11 +47,57 @@ pytest 4. Open the pull request against `dev`. 5. Promote tested release changes from `dev` to `main` through a release pull request. -## Source layout +## Architecture boundaries `src/framevitals/` is the canonical Python package. New reusable code belongs there and should import through the `framevitals.*` namespace. -The Flask API and React dashboard are optional interfaces around the package. Product logic should stay in `src/framevitals/` rather than being duplicated in application code. +The main layers have distinct responsibilities: + +| Layer | Responsibility | +| --- | --- | +| `sources.py` | Normalize inputs and expose metadata, projection, loading, and optional streaming | +| `focused.py` | Run one requested diagnostic without invoking unrelated pipeline stages | +| `analysis_api.py` | Dispatch full analysis through the appropriate source-aware execution path | +| `streaming_pipeline.py` | Orchestrate bounded full analysis for streaming-capable sources | +| `planning_api.py` | Preview execution decisions without running heavy analyses | +| `operations.py` | Cleaning, drift, contracts, validation, and quality gates | +| `result.py` / `quality_results.py` | Dict-compatible public result objects and helpers | +| `cli.py` | Thin command-line interface over canonical package APIs | +| `app.py` / `frontend/` | Optional interfaces; they must not duplicate package logic | + +A compatibility module may delegate to these layers, but it should not contain a second implementation of the same public operation. + +## Source-aware execution + +Do not call `source.load()` automatically just because an analysis accepts a file path. + +When a `StreamingDatasetSource` can safely support the operation: + +1. inspect source metadata first; +2. derive execution budgets from the **true** source shape; +3. project only required columns where practical; +4. stream or sample deterministically within the budget; +5. expose execution provenance in the returned payload. + +Approximate work must be labelled honestly. If a metric comes from a bounded row sample, do not present it as exact. Conversely, checks such as uniqueness, allowed-value validation, or hard numeric bounds must not silently become sampled checks when exactness is part of the public contract. + +## Optional dependencies + +The base package should not import optional stacks unless their feature is actually requested. + +Current capability groups include: + +- `framevitals[arrow]` for Arrow-backed streaming and Arrow-compatible table producers; +- `framevitals[duckdb]` for lazy DuckDB relations through Arrow transport; +- `framevitals[excel]` for XLS/XLSX readers; +- `framevitals[plot]` for chart/PDF plotting support; +- `framevitals[ml]` for heavier model/explainability integrations; +- `framevitals[ai]` for agentic AI integrations; +- `framevitals[web]` for the Flask runtime; +- `framevitals[docs]` for the MkDocs documentation toolchain; +- `framevitals[all]` for all optional runtime capabilities. + +If a new optional dependency is introduced, prefer a lazy import plus a clear install hint. Add a dependency-boundary test when accidental eager imports would make the base install heavier or break another extra. ## Tests @@ -49,7 +106,9 @@ Useful checks include: ```bash pytest pytest tests/test_public_api.py +pytest tests/test_public_surface_contract.py pytest tests/test_framevitals_pipeline.py +pytest tests/test_optional_dependency_boundaries.py pytest tests/test_package_boundary.py python -m compileall src/framevitals app.py python -m build @@ -57,6 +116,15 @@ python -m twine check dist/* framevitals --version ``` +For source-aware changes, also run the relevant focused suites. Examples: + +```bash +pytest tests/test_parquet_streaming.py +pytest tests/test_csv_streaming.py +pytest tests/test_streaming_drift.py +pytest tests/test_statistics_streaming.py +``` + For the optional React dashboard: ```bash @@ -65,13 +133,22 @@ npm ci npm run build ``` +Performance-sensitive changes should use the benchmark harness rather than intuition alone: + +```bash +python benchmarks/benchmark_profile_scale.py --rows 50000 --scenarios numpy auto parquet +``` + ## Style - Prefer clear Python over clever Python. - Keep the public API deliberate and small. - Avoid hidden global state in reusable library code. - Return structured, JSON-friendly values where practical. +- Preserve dict compatibility for public result objects during the 0.x series unless a breaking change is explicitly planned. - Optional analyses should fail gracefully when a dependency is unavailable. +- Keep source shape, sample shape, and materialization state distinct in execution metadata. +- Use Rust only for measured hot paths; keep orchestration and public semantics readable in Python. - Do not commit generated reports, uploads, cleaned datasets, caches, virtual environments, build output, or frontend compiler artifacts. ## Pull requests @@ -82,7 +159,8 @@ A good pull request explains: - why the chosen approach is appropriate; - what behavior changed; - what tests were run; -- any compatibility or performance implications. +- any compatibility or performance implications; +- whether the change affects exactness, sampling, optional dependencies, or public result schemas. For large features or public-API changes, open an issue first so the design can be discussed before implementation. diff --git a/README.md b/README.md index 270765e..92f969f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ### Know if your data is healthy, stable, and ML-ready — before your model finds out. -**A Python toolkit for data-quality diagnostics, drift detection, anomaly analysis, and ML-readiness checks on pandas and tabular data.** +**FrameVitals is a source-aware Python toolkit for data health, drift detection, anomaly analysis, data contracts, quality gates, and ML-readiness diagnostics on tabular data.** [![Tests](https://github.com/parthdongre/FrameVitals/actions/workflows/test.yml/badge.svg)](https://github.com/parthdongre/FrameVitals/actions/workflows/test.yml) [![PyPI](https://img.shields.io/pypi/v/framevitals.svg)](https://pypi.org/project/framevitals/) @@ -12,394 +12,316 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![GitHub stars](https://img.shields.io/github/stars/parthdongre/FrameVitals?style=social)](https://github.com/parthdongre/FrameVitals) -[Install](#installation) · [Quick start](#quick-start) · [CLI](#command-line-interface) · [Roadmap](#roadmap) · [Contributing](#contributing) +[Installation](#installation) · [Quick Start](#quick-start) · [Workflows](#common-workflows) · [CLI](#command-line) · [Docs](docs/) · [Contributing](CONTRIBUTING.md) --- -FrameVitals turns a pandas DataFrame or tabular dataset into a **structured health report** you can inspect, serialize, compare, and eventually enforce in CI. +FrameVitals helps you decide whether a dataset is **healthy enough to trust** before it reaches a model, analytics workflow, dashboard, or production pipeline. -Instead of stitching together separate profiling, quality, drift, anomaly, and ML-readiness tools, FrameVitals gives you one deliberately small entry point: +It provides one consistent API for inspecting data quality, ML readiness, anomalies, drift, contracts, validation, snapshots, and CI-friendly quality gates. ```python import framevitals as fv -report = fv.analyze(df) -drift = fv.compare(reference_df, current_df) -contract = fv.infer_contract(reference_df) -validation = fv.validate(current_df, contract) -``` - -The goal is simple: **catch bad data before it becomes a bad model, a broken dashboard, or a production incident.** - -```text - ┌──────────────────────────┐ -DataFrame / file ─► ANALYZE │ - │ profile · health · ML │ - │ stats · anomalies · risk │ - └────────────┬─────────────┘ - │ - ▼ - structured report - -Reference + current ───────────► COMPARE ─────► drift verdict +report = fv.analyze(data) +drift = fv.compare(reference, current) +contract = fv.infer_contract(reference) +validation = fv.validate(current, contract) +gate = fv.gate(current, reference=reference, contract=contract) ``` -## Why FrameVitals? - -Most data checks answer one narrow question. FrameVitals is designed around the questions that show up repeatedly in real data and ML workflows: - -| Question | FrameVitals | -| --- | --- | -| Is this dataset structurally healthy? | Missingness, duplicates, cardinality, schema and quality diagnostics | -| Is it ready for modelling? | ML-readiness scoring, target-aware checks and model diagnostics | -| Are there suspicious rows or features? | Statistical diagnostics, anomaly detection, leakage and multicollinearity checks | -| Has production data changed? | Reference-vs-current drift analysis with numeric and categorical tests | -| Can I use the result in code? | JSON-friendly structured output through a Python API and CLI | -| Will analysis unexpectedly write files? | No — filesystem artifacts are opt-in | - -FrameVitals is **package-first**. The core library lives under `src/framevitals/`; the Flask API and React dashboard are optional interfaces around the same analysis engine. +The goal is simple: **catch bad data before it becomes a bad model, broken dashboard, or production incident.** ## Installation -FrameVitals supports **Python 3.11, 3.12, and 3.13**. The current public release is **0.1.0 (alpha)** and is available on PyPI. +Install the package from PyPI: ```bash pip install framevitals ``` -Optional feature groups keep heavier dependencies out of the default install: +FrameVitals supports **Python 3.11, 3.12, and 3.13**. + +Optional capabilities are available as extras: ```bash -pip install "framevitals[ml]" # XGBoost, LightGBM, PyOD, SHAP -pip install "framevitals[ai]" # Ollama-backed AI features -pip install "framevitals[web]" # Flask web runtime -pip install "framevitals[all]" # all optional runtime features +pip install "framevitals[arrow]" # Arrow and Parquet interoperability +pip install "framevitals[duckdb]" # DuckDB relations +pip install "framevitals[plot]" # plotting and report charts +pip install "framevitals[ml]" # optional ML diagnostics +pip install "framevitals[ai]" # Ollama-backed AI capabilities +pip install "framevitals[web]" # Flask web runtime +pip install "framevitals[all]" # all optional runtime capabilities ``` -## Quick start +## Quick Start -### Analyze a DataFrame +Analyze a file directly: ```python -import pandas as pd import framevitals as fv -customers = pd.read_csv("customers.csv") -report = fv.analyze(customers) +report = fv.analyze("customers.csv") -print(report["health"]["overall_score"]) -print(report["ml_readiness"]) +print(report.health["overall_score"]) +print(report.ml_readiness) +print(report.findings[:3]) ``` -File paths work too: +Or pass a pandas DataFrame: ```python -report = fv.analyze("customers.csv", mode="quick") +import pandas as pd +import framevitals as fv + +customers = pd.read_csv("customers.csv") +report = fv.analyze(customers) ``` -FrameVitals supports pandas DataFrames and common tabular file formats including CSV, TSV, Excel, and JSON. +FrameVitals also supports Parquet, PyArrow data, and lazy DuckDB relations when the corresponding optional dependencies are installed. -### Add a supervised-learning target +## Common Workflows -```python -report = fv.analyze( - customers, - target="churn", - mode="deep", -) +### Run only the diagnostic you need -print(report["model_leaderboard"]) -print(report["explainability"]) -``` +The focused APIs let you inspect one part of a dataset without running the complete analysis pipeline: -Target-aware analysis can surface modelling risks such as leakage, imbalance, redundant features, unstable relationships, and weak baselines. +```python +fv.profile(data) +fv.health(data) +fv.quality(data) +fv.ml_readiness(data) +fv.statistics(data) +fv.anomalies(data) +fv.relationships(data) +``` ### Compare datasets for drift ```python -reference = pd.read_csv("training_data.csv") -current = pd.read_csv("production_batch.csv") - result = fv.compare(reference, current) -print(result["summary"]["overall_verdict"]) +print(result.severity) print(result["columns"][:3]) ``` -Numeric drift uses **PSI, Kolmogorov-Smirnov statistics, and standardized mean shift**. Categorical drift uses **PSI and chi-square diagnostics**. +Use this to compare training and production data, historical batches, pipeline outputs, or any reference/current pair. -### Validate a data contract - -Infer a contract once from a trusted reference dataset, then validate later -batches before they reach downstream jobs: +### Infer and validate a data contract ```python contract = fv.infer_contract(reference) result = fv.validate(current, contract) -if not result["valid"]: - for finding in result["errors"]: +if result.status == "fail": + for finding in result.findings: print(finding["message"]) ``` -Contracts capture required columns, broad data types, nullability, and finite -numeric bounds. They are plain JSON-friendly dictionaries, so a contract can -be committed with a pipeline or stored with a dataset baseline. +Contracts can capture expectations such as schema, data types, nullability, numeric bounds, allowed values, and uniqueness. -## The public API +### Add a quality gate -The public API is intentionally small while FrameVitals is in alpha. +```python +result = fv.gate( + current, + reference=reference, + contract=contract, +) -| API | Status | Purpose | -| --- | --- | --- | -| `framevitals.analyze(...)` | Available in `0.1.0` | Profile and diagnose one dataset | -| `framevitals.compare(...)` | Available in `0.1.0` | Compare reference and current data for drift | -| `framevitals.infer_contract(...)` | Available on `dev` | Infer a reusable data contract from reference data | -| `framevitals.validate(...)` | Available on `dev` | Validate data against an inferred or explicit contract | -| snapshots / monitoring | Roadmap | Reuse baselines for recurring schema and drift checks | +print(result.status) # pass / warn / fail +print(result.passed) +``` -This keeps the library easy to learn while leaving room for the result model and validation system to mature before `1.0`. +A gate combines the checks you choose into one verdict that can be used in scripts, pipelines, and CI. -## What FrameVitals checks +### Add domain-specific checks -| Area | Examples | -| --- | --- | -| **Structure** | shape, dtypes, semantic column roles, date/text detection | -| **Data quality** | missingness, duplicates, constants, cardinality, outliers | -| **Health scoring** | overall dataset health plus component-level diagnostics | -| **ML readiness** | modelling readiness, risky columns, preprocessing recommendations | -| **Statistics** | distribution checks, normality, correlations, effect-size style diagnostics | -| **Anomalies** | multivariate and robust outlier detectors, optional ensemble methods | -| **Target intelligence** | task inference, leakage hints, multicollinearity, feature/model diagnostics | -| **Drift** | PSI, KS, chi-square, mean shift, new or disappearing categories | -| **Time series** | date-aware diagnostics, stationarity, decomposition and forecast previews | -| **Text** | text-column profiling, vocabulary and lightweight semantic diagnostics | -| **Explainability** | model feature importance and SHAP when the optional ML stack is installed | +```python +@fv.check("positive revenue", severity="error") +def positive_revenue(df): + return { + "passed": bool((df["revenue"] >= 0).all()), + "message": "Negative revenue values were found.", + } + +result = fv.gate(data, custom_checks=[positive_revenue]) +``` -Not every analysis runs on every dataset. FrameVitals uses dataset signals, selected mode, target availability, and installed optional dependencies to decide what is useful and safe to execute. +Custom checks make it possible to enforce application-specific rules without modifying FrameVitals itself. -## Analysis modes +### Run target-aware analysis ```python -fv.analyze(df, mode="quick") -fv.analyze(df, mode="standard") -fv.analyze(df, mode="deep") -fv.analyze(df, mode="research") +report = fv.analyze( + data, + target="churn", + mode="deep", +) ``` -| Mode | Best for | -| --- | --- | -| `quick` | Fast structural, quality, and ML-readiness checks | -| `standard` | Everyday analysis with broader diagnostics | -| `deep` | Target-aware and heavier statistical analysis | -| `research` | Largest analysis budget for exploratory work | +Target-aware analysis can surface modelling risks such as leakage, imbalance, redundant features, multicollinearity, and weak baseline relationships. + +### Create monitoring snapshots -## Filesystem artifacts are opt-in +```python +report = fv.analyze(current) +snapshot = report.snapshot("snapshot.json") +``` -FrameVitals is designed to behave like a library first. Calling the Python API does not need to scatter reports and cleaned files around your working directory. +Compare compact snapshots later without retaining every raw dataset: ```python -report = fv.analyze(df) -assert report["cleaning"]["output_path"] is None +previous = fv.load_snapshot("previous.json") +latest = fv.load_snapshot("snapshot.json") +change = fv.compare_snapshots(previous, latest) +``` -report = fv.analyze(df, artifacts=True) -print(report["cleaning"]["output_path"]) +## Analysis Modes + +Choose how much work FrameVitals should perform: + +```python +fv.analyze(data, mode="quick") +fv.analyze(data, mode="standard") +fv.analyze(data, mode="deep") +fv.analyze(data, mode="research") ``` -## Command-line interface +Use `quick` for fast checks and the deeper modes when you want broader statistical or modelling diagnostics. -FrameVitals also ships with a CLI for scripts, terminals, and future CI workflows. +## Source-Aware Execution -Start by discovering the available commands and options: +FrameVitals is designed to work with more than pandas alone. Supported sources can include DataFrames, files, Arrow-native data, and DuckDB relations. -```bash -framevitals --help -framevitals analyze --help -framevitals compare --help -framevitals --version -``` +Where semantics allow it, large or lazy sources can use bounded or streaming execution instead of being loaded fully into pandas. Operations that require exact results can still materialize the full dataset, and execution metadata reports those decisions. + +## Command Line + +The Python package also includes a CLI. Analyze a dataset: ```bash -framevitals analyze dataset.csv -framevitals analyze dataset.csv --mode quick -framevitals analyze dataset.csv --target churn --mode deep -framevitals analyze dataset.csv --output report.json -framevitals analyze dataset.csv --artifacts +framevitals analyze customers.csv ``` -A useful end-to-end smoke test is: +Compare two datasets: ```bash -framevitals analyze dataset.csv --target churn --mode deep --artifacts --output report.json +framevitals compare reference.csv current.csv ``` -Compare two datasets for drift: +Infer a contract: ```bash -framevitals compare train.csv production.csv -framevitals compare train.csv production.csv --columns age,income -framevitals compare train.csv production.csv --output drift.json +framevitals infer-contract reference.csv ``` -Create and use a contract from the terminal: +Create a monitoring snapshot: ```bash -framevitals infer-contract training_data.csv --output contract.json -framevitals validate production_batch.csv --contract contract.json +framevitals snapshot customers.csv ``` -The validation command exits with status `1` when the contract has errors, -which makes it suitable for CI jobs and scheduled ingestion checks. - -In `0.1.0`, `framevitals analyze` prints a compact analysis summary to the terminal. `--output report.json` writes that CLI summary to the requested path. `--artifacts` enables generated files in the current working directory, including a cleaned dataset under `cleaned/` and generated charts under `static/charts/`. The full structured analysis result is available through the Python API with `framevitals.analyze(...)`. - -## Optional ML and AI features - -The default package contains the core data-health engine. Heavier features are separated into extras so a simple install stays predictable. +Inspect dataset execution capabilities: ```bash -pip install "framevitals[ml]" +framevitals inspect customers.csv ``` -Adds optional integrations including XGBoost, LightGBM, PyOD and SHAP. +See all commands and options with: ```bash -pip install "framevitals[ai]" +framevitals --help ``` -Adds Ollama-backed interpretation and question-answering features. AI is treated as an optional explanation layer; computed diagnostics remain usable without a reachable model. - -## Web dashboard +## CI and GitHub Actions -The repository includes an optional **Flask API + React/TypeScript dashboard** for interactive exploration. +FrameVitals can sit between your data pipeline and downstream work: -```bash -pip install -e ".[web]" -python app.py +```text +Data / ETL + ↓ +FrameVitals Gate + ↓ +PASS / WARN / FAIL + ↓ +Training / Analytics / Production ``` -Then in another terminal: +The repository includes a reusable GitHub Action: -```bash -cd frontend -npm ci -npm run dev +```yaml +- uses: parthdongre/FrameVitals@v0.2.0 + id: framevitals + with: + current: data/production.parquet + reference: data/training.parquet + contract: data/contract.json + output: framevitals-gate.json ``` -Typical local endpoints: - -- Flask API: `http://127.0.0.1:5055` -- React dashboard: `http://127.0.0.1:5173` +For production workflows, pin the action to a released tag or commit. -The public project website will remain separate from the package runtime so the library does not depend on a hosted service. +## Python API -## Design principles +The main workflow entry points are available directly from `framevitals`: -FrameVitals is being built around a few constraints that are easy to lose in analytics projects: - -- **DataFrame first** — use it directly from Python without routing through a web app. -- **Structured results** — return reusable data, not only screenshots or prose. -- **Safe defaults** — no unexpected artifact writes and graceful optional-feature fallbacks. -- **Small public API** — make the common path obvious before exposing every internal module. -- **Optional heavy dependencies** — ML, AI, and web features should not bloat a basic install. -- **Production direction** — drift, contracts, snapshots, and CI quality gates are first-class roadmap items. - -## Project layout - -```text -. -├── src/framevitals/ # canonical installable Python package -├── tests/ # automated test suite -├── frontend/ # optional React + TypeScript dashboard -├── templates/ # Flask report pages -├── static/ # web/report assets -├── app.py # optional Flask API/server -├── pyproject.toml # package metadata and dependency groups -└── .github/workflows/ # CI, package validation and publishing +```python +fv.analyze(...) +fv.plan(...) +fv.profile(...) +fv.health(...) +fv.quality(...) +fv.ml_readiness(...) +fv.statistics(...) +fv.anomalies(...) +fv.relationships(...) +fv.compare(...) +fv.infer_contract(...) +fv.validate(...) +fv.check(...) +fv.run_checks(...) +fv.gate(...) +fv.create_snapshot(...) +fv.compare_snapshots(...) ``` -New reusable Python code belongs in `src/framevitals/` and should import through the `framevitals.*` namespace. +For detailed API behaviour, configuration, source semantics, performance notes, and advanced usage, see [`docs/`](docs/). ## Development +Clone the repository and install it in development mode: + ```bash git clone https://github.com/parthdongre/FrameVitals.git cd FrameVitals -git switch dev - -python -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip pip install -e ".[all,dev]" - -pytest -python -m build -python -m twine check dist/* ``` -On Windows PowerShell: - -```powershell -.venv\Scripts\Activate.ps1 -``` +Run the test suite: -CI validates the core package across Python 3.11–3.13, optional features, the React build, wheel contents, distribution metadata, and a clean-wheel install. - -Development is integrated through `dev`; `main` is kept release-ready. - -## Roadmap - -FrameVitals is moving toward a complete data-health quality gate: - -```text -0.1 ANALYZE + COMPARE - data health · ML readiness · target diagnostics · drift - -0.2 VALIDATE + SNAPSHOTS - data contracts · CI gates · reusable baselines - -0.3 RESULT OBJECTS + ADVANCED DRIFT - stronger result model · large-data handling · richer monitoring - -0.4 EXTENSIBILITY + INTEGRATIONS - configurable checks · adapters · monitoring workflows - -1.0 STABLE DATA-HEALTH API - dependable analyze → compare → validate → monitor workflow +```bash +pytest ``` -Near-term work is tracked through issues and the `dev` branch. - -## Project status - -FrameVitals `0.1.x` is **alpha software**. The core API is usable, but the project is intentionally still refining naming, result schemas, thresholds, and extension points before `1.0`. - -If you are using FrameVitals in a project, feedback about real datasets, false positives, missing diagnostics, performance, and API ergonomics is especially valuable. - ## Contributing -Contributions are welcome. +Contributions are welcome, including bug fixes, diagnostics, tests, documentation, integrations, and performance improvements. -A good contribution is focused, tested, and improves either the reliability of a diagnostic or the clarity of the public workflow. +Please read [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request. -Start with [CONTRIBUTING.md](CONTRIBUTING.md), and please read the [Code of Conduct](CODE_OF_CONDUCT.md) and [Security Policy](SECURITY.md). +## Documentation -## Releases +Detailed documentation lives in [`docs/`](docs/). -Releases are built and validated in GitHub Actions and published through PyPI Trusted Publishing. See [RELEASING.md](RELEASING.md) and [CHANGELOG.md](CHANGELOG.md). +- [Changelog](CHANGELOG.md) +- [Contributing Guide](CONTRIBUTING.md) +- [Issue Tracker](https://github.com/parthdongre/FrameVitals/issues) ## License -FrameVitals is open source under the [MIT License](LICENSE). - ---- - -
- -**If FrameVitals is useful to you, consider starring the repository — it helps the project grow.** - -
+FrameVitals is released under the [MIT License](LICENSE). diff --git a/RELEASING.md b/RELEASING.md index 4b21ff5..6505253 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,44 +1,94 @@ # Releasing FrameVitals -FrameVitals is configured to publish to PyPI through GitHub Actions Trusted Publishing. No long-lived PyPI API token is required. +FrameVitals publishes to PyPI through GitHub Actions Trusted Publishing. No long-lived PyPI API token is required. ## Branch model -- `main` is the release-ready branch. Published releases and version tags come from `main`. -- `dev` is the integration branch for ongoing development. -- Feature and dependency-update pull requests should target `dev`. -- When a release is ready, promote the tested release changes from `dev` to `main`, update the version/changelog, and publish from `main`. +- `main` is release-ready; version tags and published GitHub releases come from `main`. +- `dev` is the normal integration branch. +- Stabilization branches such as `develop/august` are promoted into `dev` only after their contracts and specialized CI lanes are clean. +- A stabilization-branch merge is not itself a PyPI release. -## One-time PyPI setup +## Release candidate checklist + +Before promoting a release candidate toward `main`: + +1. Core Tests, Package, minimum-dependency, platform, Arrow/fallback, native Rust, optional-feature, frontend/package-quality, docs, and security-relevant CI must be green. +2. Performance-sensitive changes must have benchmark evidence and accuracy/safety checks; do not publish unsupported universal speed claims. +3. Review generated files, temporary artifacts, stale compatibility shims, and branch-only experiments before promotion. +4. Update `CHANGELOG.md` with the release section and any known 0.x compatibility changes. +5. Keep all release-version sources synchronized: + - GitHub release tag, e.g. `v0.2.0`; + - `pyproject.toml` → `[project].version`; + - `src/framevitals/__init__.py` → `__version__`; + - `rust/framevitals-core/Cargo.toml` → `[package].version`; + - `rust/framevitals-py/Cargo.toml` → `[package].version`. +6. Confirm the package workflow builds and validates both the portable fallback wheel and a compatible native wheel. +7. Confirm the clean-environment package smoke proves pip prefers the native wheel when both compatible native and universal fallback wheels are present. + +## Distribution model + +FrameVitals intentionally publishes two kinds of Python wheels: + +- **Native ABI3 wheels** for supported/common platforms. These include `framevitals._native` and are preferred automatically by pip when compatible. +- **Portable fallback wheel** (`py3-none-any`) for environments without a published native wheel. + +The release workflow also publishes one source distribution. -Before the first release: +Current native release targets are: -1. Create or sign in to your PyPI account. -2. Open your PyPI account's **Publishing** page and configure a pending Trusted Publisher for project name `framevitals`. +- Linux x86_64; +- macOS arm64; +- macOS x86_64; +- Windows x86_64. + +The native extension uses a Python 3.11 ABI3 floor so one compatible wheel can serve supported Python 3.11+ versions on the same platform. + +## One-time PyPI setup + +1. Create/sign in to the PyPI account. +2. Configure a pending Trusted Publisher for project `framevitals`. 3. GitHub owner: `parthdongre`. 4. Repository: `FrameVitals`. 5. Workflow filename: `release.yml`. 6. GitHub environment: `pypi`. -7. In GitHub repository settings, create a `pypi` environment. Requiring manual approval for production publishing is recommended. +7. Create the matching `pypi` environment in GitHub repository settings; production approval is recommended. + +If the repository is renamed, update the PyPI Trusted Publisher configuration before publishing again. + +## Publish checklist + +1. Promote the reviewed release candidate to `main` through the normal branch process. +2. Verify the exact `main` commit has the intended release versions and changelog. +3. Create a GitHub release from that exact commit with a matching tag such as `v0.2.0`. +4. Publishing the GitHub release triggers `.github/workflows/release.yml`. +5. The workflow must: + - verify tag/Python/Rust version consistency; + - build one fallback wheel and one source distribution; + - build the configured native ABI3 wheels; + - verify each native wheel actually contains `framevitals._native`; + - run `twine check` across the complete payload; + - publish through Trusted Publishing only after all build jobs pass. +6. Confirm the expected files appear on PyPI. +7. In a brand-new environment, run: -A pending publisher does not reserve the package name until the first successful publication. If the repository is renamed later, update the PyPI Trusted Publisher configuration before publishing again. + ```bash + python -m pip install --upgrade framevitals + python -c "import framevitals; print(framevitals.__version__)" + framevitals --version + ``` -## Release checklist +8. On a supported native platform, also verify: -1. Confirm the release commit is on `main` and the Tests and Package workflows are green. -2. Confirm `CHANGELOG.md` contains the release notes and date. -3. Confirm `pyproject.toml` and `src/framevitals/__init__.py` contain the same release version. -4. Confirm the built wheel imports successfully and `framevitals --version` reports the release version. -5. Create a GitHub release from `main` with a matching tag such as `v0.1.0`. -6. Publishing the GitHub release triggers `.github/workflows/release.yml`. -7. Confirm the PyPI publish workflow succeeds. -8. Install the published package in a clean environment and run `framevitals --version`. + ```bash + python -c "from framevitals.backends import backend_status; print(backend_status())" + ``` -## Version consistency + `native_available` should be `True` and automatic backend selection should choose `rust`. +9. Smoke-test one normal analysis and one source-aware/Arrow path from the published wheel. -Until version metadata is centralized, keep these two values identical: +## Release discipline -- `pyproject.toml` → `[project].version` -- `src/framevitals/__init__.py` → `__version__` +Do not bump versions merely to merge a stabilization branch into `dev`. Version bumps belong to an intentional release-candidate commit. -A future cleanup can derive the package version from one canonical source. +Do not publish a release because the feature list is long. Publish when correctness, packaging, provenance, platform compatibility, and performance-sensitive behavior have all passed their release gates. diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..4995e30 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,97 @@ +# FrameVitals support + +FrameVitals is alpha software and the fastest way to fix a problem is a small, +reproducible report that shows **what input/source path was used, which optional +capabilities were installed, and what FrameVitals actually executed**. + +## Before opening an issue + +1. Reproduce the problem on the newest compatible FrameVitals release or the relevant + development branch. +2. Reduce the input to the smallest dataset that still reproduces the behavior. +3. Check whether the same call behaves differently with a pandas DataFrame versus the + original source adapter. +4. Include execution/source metadata when the problem involves memory, sampling, + streaming, or performance. + +Useful commands: + +```bash +framevitals --version +framevitals system-info --no-probe-gpu --format json +framevitals inspect path/to/dataset.parquet --format json +``` + +If the problem specifically involves CUDA/GPU probing, run `framevitals system-info` +without `--no-probe-gpu` as well. + +## What to include in a bug report + +Please provide: + +- FrameVitals version or commit SHA; +- Python version and operating system; +- the exact command or Python call; +- the smallest reproducible input shape/schema; +- which optional extras are installed (`arrow`, `duckdb`, `excel`, `plot`, `ml`, `ai`, + or `web`); +- `framevitals system-info --no-probe-gpu --format json` output when relevant; +- `framevitals inspect ... --format json` output for source/streaming issues; +- the complete exception traceback; +- expected behavior and actual behavior. + +For a focused diagnostic, include its execution metadata when possible: + +```python +result = fv.statistics(data, mode="quick") +print(result.execution) +``` + +This tells maintainers whether the operation sampled, streamed, or fully materialized +its source. + +## Reproduction datasets + +Do **not** upload confidential, personal, regulated, proprietary, or customer data to a +public issue. Prefer one of these: + +- a tiny synthetic DataFrame; +- a short CSV created specifically for the reproduction; +- code that deterministically generates the failing data pattern; +- schema/source metadata with sensitive values removed. + +## Performance reports + +For performance regressions, include: + +- dataset rows/columns and approximate file size; +- source format and whether `inspect_source()` reports streaming/projection support; +- analysis mode; +- `result.execution` when available; +- wall-clock time and peak memory if measured; +- whether `FRAMEVITALS_BACKEND` was configured explicitly; +- native-core availability from `system-info`. + +The repository also contains reproducible benchmark and catastrophic-regression +workflows under `benchmarks/` and `.github/workflows/`. + +## Feature requests + +A strong feature request explains the workflow problem before proposing a new module. +FrameVitals deliberately prefers a small analyze → compare → validate → gate → monitor +surface over accumulating unrelated analytics features. + +For source integrations, first check whether the producer already supports the Arrow C +Stream / PyCapsule protocol; a standard interoperability boundary is preferable to a +library-specific dependency when it preserves the required semantics. + +## Security issues + +Do not report vulnerabilities, credential exposure, or sensitive-data security issues +through a normal public support thread. Follow the repository's `SECURITY.md` process. + +## General questions + +Questions that can become durable documentation are welcome. When possible, include a +small code example and the desired outcome so the answer can be turned into a docs or +example improvement for future users. diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..ab45c5a --- /dev/null +++ b/action.yml @@ -0,0 +1,138 @@ +name: FrameVitals Data Quality Gate +description: Run FrameVitals drift and contract checks as a CI quality gate. +author: Parth Dongre + +branding: + icon: activity + color: blue + +inputs: + current: + description: Path to the current dataset in the caller workspace. + required: true + reference: + description: Optional reference dataset path for drift checks. + required: false + default: "" + contract: + description: Optional FrameVitals contract JSON path for exact validation. + required: false + default: "" + columns: + description: Optional comma-separated columns to include in drift checks. + required: false + default: "" + max-columns: + description: Maximum shared columns to compare for drift. + required: false + default: "30" + drift-warn-on: + description: Drift severity that changes a passing gate to warning. + required: false + default: moderate + drift-fail-on: + description: Drift severity that fails the gate. + required: false + default: severe + fail-on-validation-warning: + description: Set to true to promote contract warnings to failures. + required: false + default: "false" + output: + description: JSON result path written in the caller workspace. + required: false + default: framevitals-gate.json + python-version: + description: Python version used to run FrameVitals. + required: false + default: "3.12" + +outputs: + status: + description: FrameVitals gate status (pass, warn, or fail). + value: ${{ steps.gate.outputs.status }} + passed: + description: Whether the gate avoided a failing verdict. + value: ${{ steps.gate.outputs.passed }} + result-path: + description: Path to the JSON gate result. + value: ${{ steps.gate.outputs.result-path }} + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + + - name: Install FrameVitals action runtime + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + run: | + python -m pip install --upgrade pip + cd "$ACTION_PATH" + python -m pip install ".[arrow]" + + - name: Run FrameVitals quality gate + id: gate + shell: bash + env: + CURRENT: ${{ inputs.current }} + REFERENCE: ${{ inputs.reference }} + CONTRACT: ${{ inputs.contract }} + COLUMNS: ${{ inputs.columns }} + MAX_COLUMNS: ${{ inputs.max-columns }} + DRIFT_WARN_ON: ${{ inputs.drift-warn-on }} + DRIFT_FAIL_ON: ${{ inputs.drift-fail-on }} + FAIL_ON_VALIDATION_WARNING: ${{ inputs.fail-on-validation-warning }} + OUTPUT: ${{ inputs.output }} + run: | + if [[ -z "$REFERENCE" && -z "$CONTRACT" ]]; then + echo "FrameVitals gate requires at least one of reference or contract." >&2 + exit 2 + fi + + args=( + gate "$CURRENT" + --max-columns "$MAX_COLUMNS" + --drift-warn-on "$DRIFT_WARN_ON" + --drift-fail-on "$DRIFT_FAIL_ON" + --format json + --output "$OUTPUT" + ) + + if [[ -n "$REFERENCE" ]]; then + args+=(--reference "$REFERENCE") + fi + if [[ -n "$CONTRACT" ]]; then + args+=(--contract "$CONTRACT") + fi + if [[ -n "$COLUMNS" ]]; then + args+=(--columns "$COLUMNS") + fi + if [[ "$FAIL_ON_VALIDATION_WARNING" == "true" ]]; then + args+=(--fail-on-validation-warning) + fi + + set +e + framevitals "${args[@]}" + exit_code=$? + set -e + + python - "$OUTPUT" "$GITHUB_OUTPUT" <<'PY' + import json + import sys + from pathlib import Path + + result_path = Path(sys.argv[1]) + github_output = Path(sys.argv[2]) + payload = json.loads(result_path.read_text(encoding="utf-8")) + with github_output.open("a", encoding="utf-8") as handle: + handle.write(f"status={payload.get('status', 'unknown')}\n") + handle.write(f"passed={str(bool(payload.get('passed'))).lower()}\n") + handle.write(f"result-path={result_path}\n") + PY + + exit "$exit_code" diff --git a/app.py b/app.py index d022c28..f202231 100644 --- a/app.py +++ b/app.py @@ -5,39 +5,21 @@ server-rendered report routes available for local use. """ -import os import math -from pathlib import Path +import os from copy import deepcopy -from time import perf_counter +from pathlib import Path from threading import Lock, Thread +from time import perf_counter -from flask import Flask, render_template, request, redirect, url_for, send_file, session, jsonify +from flask import Flask, jsonify, redirect, render_template, request, send_file, session, url_for from werkzeug.exceptions import ClientDisconnected -from framevitals.loader import ( - load_dataset, - save_uploaded_file, -) -from framevitals.pipeline import ( - run_full_analysis, -) -from framevitals.ai_insights import ( - answer_dataset_question, -) -from framevitals.ai_agent import ( - answer_with_agent, -) -from framevitals.report_generator import ( - generate_pdf_report, -) -from framevitals.frontend_api import ( - build_dashboard_payload, -) -from framevitals.drift_analysis import ( - compare_datasets, - split_by_date, -) +from framevitals.ai_insights import answer_dataset_question +from framevitals.drift_analysis import split_by_date +from framevitals.frontend_api import build_dashboard_payload +from framevitals.loader import load_dataset, save_uploaded_file +from framevitals.pipeline import run_full_analysis # --------------------------------------------------------------------------- @@ -48,7 +30,6 @@ # JSON.parse both do), which makes the frontend silently fall back to an # empty payload. We walk every payload recursively and replace those values # with None so the wire format is RFC-8259 compliant. - def _is_nonfinite(v) -> bool: return isinstance(v, float) and not math.isfinite(v) @@ -73,7 +54,7 @@ def safe_jsonify(payload): app = Flask(__name__) app.secret_key = os.environ.get( "FRAMEVITALS_SECRET_KEY", - "development-only-secret" + "development-only-secret", ) app.config["MAX_CONTENT_LENGTH"] = 50 * 1024 * 1024 @@ -90,6 +71,7 @@ def safe_jsonify(payload): class DotDict(dict): """Allow dict.key access for Jinja templates.""" + __getattr__ = dict.get __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ @@ -112,7 +94,12 @@ def _get_report_job(dataset_id: str) -> dict: return dict(REPORT_JOBS.get(dataset_id, {})) -def _set_report_job(dataset_id: str, status: str, pdf_path: Path | None = None, error: str | None = None) -> dict: +def _set_report_job( + dataset_id: str, + status: str, + pdf_path: Path | None = None, + error: str | None = None, +) -> dict: job = { "status": status, "pdf_path": str(pdf_path) if pdf_path else None, @@ -142,6 +129,10 @@ def _queue_pdf_generation(dataset_id: str, result: dict | None = None) -> dict: def worker(): _set_report_job(dataset_id, "running") try: + # PDF/report dependencies are intentionally optional for the Flask + # runtime. Import them only when a report is actually requested. + from framevitals.report_generator import generate_pdf_report + pdf_path = generate_pdf_report(deepcopy(cached_result)) _set_report_job(dataset_id, "ready", pdf_path=pdf_path) except Exception as exc: @@ -198,7 +189,10 @@ def analyze(): except ClientDisconnected: return render_template( "error.html", - message="The upload was interrupted before Flask finished reading the file. Please try again.", + message=( + "The upload was interrupted before Flask finished reading the file. " + "Please try again." + ), ), 400 if not uploaded_file or uploaded_file.filename == "": @@ -220,7 +214,7 @@ def analyze(): with REPORT_LOCK: ANALYSIS_CACHE[dataset_id] = deepcopy(result) - report_job = _queue_pdf_generation(dataset_id, result) + _queue_pdf_generation(dataset_id, result) result["report_status"] = _report_status_payload(dataset_id) session["dataset_id"] = dataset_id @@ -229,13 +223,12 @@ def analyze(): session["analysis_mode"] = analysis_mode session["target_column"] = target_column - # Convert to DotDict for Jinja template dot-access result_dot = DotDict.from_dict(result) - return render_template("report.html", result=result_dot) except Exception as exc: import traceback + traceback.print_exc() return render_template("error.html", message=str(exc)) @@ -249,7 +242,12 @@ def api_analyze(): analysis_mode = request.form.get("analysis_mode", "standard") target_column = request.form.get("target_column") or None except ClientDisconnected: - return jsonify({"error": "The upload was interrupted before the server finished reading it. Please try again."}), 400 + return jsonify({ + "error": ( + "The upload was interrupted before the server finished reading it. " + "Please try again." + ) + }), 400 if analysis_mode not in {"quick", "standard", "deep", "research"}: analysis_mode = "standard" @@ -331,9 +329,11 @@ def ask(): _queue_pdf_generation(dataset_id, result) result["report_status"] = _report_status_payload(dataset_id) - # Try agentic answer first; fall back to the legacy single-shot answerer - # if anything goes wrong (Ollama offline, model errors, etc.). + # Agentic AI is an optional capability. If it is not installed or the + # model is unavailable, fall back to the lightweight answerer. try: + from framevitals.ai_agent import answer_with_agent + agent_response = answer_with_agent( question=question, df=load_dataset(Path(file_path)), @@ -375,7 +375,6 @@ def api_ask(): if not question: return jsonify({"error": "Missing 'question' in request body."}), 400 - # Look up the cached analysis result first with REPORT_LOCK: cached_result = ANALYSIS_CACHE.get(dataset_id) @@ -384,7 +383,6 @@ def api_ask(): "error": "No cached analysis was found for this dataset. Run /api/analyze first.", }), 404 - # Reload dataframe (cheap; uploaded files live under uploads/) file_path = session.get("file_path") df = None if file_path: @@ -394,9 +392,8 @@ def api_ask(): df = None try: - # Fast mode by default (single writer call, no critic/repair). - # Pass {"mode": "full"} in the body to opt back in to the full - # planner→executor→critic→writer loop. + from framevitals.ai_agent import answer_with_agent + mode = (body.get("mode") or "fast").lower().strip() response = answer_with_agent( question=question, @@ -413,7 +410,11 @@ def api_ask(): ml_readiness=cached_result["ml_readiness"], advanced=cached_result.get("advanced"), ) - response = {"source": response.get("source", "fallback"), "answer": response.get("answer", str(exc)), "trace": {}} + response = { + "source": response.get("source", "fallback"), + "answer": response.get("answer", str(exc)), + "trace": {}, + } return safe_jsonify({ "question": question, @@ -425,6 +426,7 @@ def api_ask(): except Exception as exc: import traceback + traceback.print_exc() return jsonify({"error": str(exc)}), 500 @@ -450,11 +452,11 @@ def api_ai_report(): cached = ANALYSIS_CACHE.get(dataset_id) if cached is None: - return jsonify({"error": "No cached analysis for this dataset. Re-run /api/analyze."}), 404 + return jsonify({ + "error": "No cached analysis for this dataset. Re-run /api/analyze." + }), 404 - from framevitals.ai_insights import ( - generate_ai_report, - ) + from framevitals.ai_insights import generate_ai_report try: ai_report = generate_ai_report( @@ -469,7 +471,6 @@ def api_ai_report(): except Exception as exc: ai_report = {"source": f"error: {exc}", "text": str(exc)} - # Re-cache so subsequent /api/analyze calls (or PDF regen) include it. with REPORT_LOCK: cached["ai_report"] = ai_report ANALYSIS_CACHE[dataset_id] = cached @@ -478,6 +479,7 @@ def api_ai_report(): except Exception as exc: import traceback + traceback.print_exc() return jsonify({"error": str(exc)}), 500 @@ -539,19 +541,23 @@ def api_compare(): _, ref_path, _ = save_uploaded_file(ref_file) _, cur_path, _ = save_uploaded_file(cur_file) - df_ref = load_dataset(ref_path) - df_cur = load_dataset(cur_path) - columns_param = request.form.get("columns", "").strip() - columns = [c.strip() for c in columns_param.split(",") if c.strip()] if columns_param else None + columns = ( + [c.strip() for c in columns_param.split(",") if c.strip()] + if columns_param + else None + ) - report = compare_datasets(df_ref, df_cur, columns=columns) + from framevitals.operations import compare + + report = compare(ref_path, cur_path, columns=columns) report["reference_filename"] = ref_file.filename report["current_filename"] = cur_file.filename return safe_jsonify(report) except Exception as exc: import traceback + traceback.print_exc() return jsonify({"error": str(exc)}), 500 @@ -592,7 +598,9 @@ def api_compare_self(): except ValueError as exc: return jsonify({"error": str(exc)}), 400 - report = compare_datasets(df_ref, df_cur) + from framevitals.operations import compare + + report = compare(df_ref, df_cur) report["reference_filename"] = f"{ds_name} (older {ratio:.0%})" report["current_filename"] = f"{ds_name} (newer {1 - ratio:.0%})" report["split_by"] = date_column @@ -601,6 +609,7 @@ def api_compare_self(): except Exception as exc: import traceback + traceback.print_exc() return jsonify({"error": str(exc)}), 500 @@ -639,11 +648,17 @@ def download_report(dataset_id): if report_status["ready"] and pdf_path.exists() and pdf_path.stat().st_size > 0: return send_file(pdf_path, as_attachment=True) - message = "The PDF report is generating in the background. Please try again in a few seconds." + message = ( + "The PDF report is generating in the background. " + "Please try again in a few seconds." + ) if report_status["status"] == "failed": message = f"PDF generation failed: {report_status.get('error', 'Unknown error')}" elif result is None: - message = "No cached analysis was found for this dataset. Please run analysis again first." + message = ( + "No cached analysis was found for this dataset. " + "Please run analysis again first." + ) return render_template("error.html", message=message), 202 diff --git a/benchmarks/benchmark_deep_pipeline.py b/benchmarks/benchmark_deep_pipeline.py new file mode 100644 index 0000000..4026f04 --- /dev/null +++ b/benchmarks/benchmark_deep_pipeline.py @@ -0,0 +1,122 @@ +"""Reproducible 8k x 180 Deep pipeline benchmark. + +This workload is the stable performance target for the August optimization +track. It is intentionally numeric and wide so profiling, adaptive deep +statistics, anomaly screening, and pairwise diagnostics all receive work. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import time + +import numpy as np +import pandas as pd + +from framevitals.pipeline import run_full_analysis + + +DEFAULT_ROWS = 8_000 +DEFAULT_COLUMNS = 180 +HISTORICAL_ORIGINAL_SECONDS = 76.17 +TEN_X_TARGET_SECONDS = HISTORICAL_ORIGINAL_SECONDS / 10.0 + + +def make_workload( + rows: int = DEFAULT_ROWS, + columns: int = DEFAULT_COLUMNS, + *, + seed: int = 42, +) -> pd.DataFrame: + if rows < 100 or columns < 32: + raise ValueError("benchmark requires at least 100 rows and 32 columns") + + rng = np.random.default_rng(seed) + values = rng.normal(size=(rows, columns)) + + # Make a bounded set of columns diagnostically interesting so deep triage + # consistently exercises skew, missingness, constants, and relationships. + skewed = min(12, columns) + values[:, :skewed] = rng.lognormal( + mean=0.0, + sigma=1.5, + size=(rows, skewed), + ) + + for index in range(12, min(24, columns)): + values[::11, index] = np.nan + + for index in range(24, min(28, columns)): + values[:, index] = float(index) + + for offset, index in enumerate(range(30, min(40, columns))): + source = offset % min(10, columns) + values[:, index] = values[:, source] * 0.97 + rng.normal( + scale=0.03, + size=rows, + ) + + return pd.DataFrame( + values, + columns=[f"n{index:03d}" for index in range(columns)], + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--rows", type=int, default=DEFAULT_ROWS) + parser.add_argument("--columns", type=int, default=DEFAULT_COLUMNS) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + os.environ.setdefault("FRAMEVITALS_BACKEND", "numpy") + frame = make_workload(args.rows, args.columns) + + started = time.perf_counter() + result = run_full_analysis( + "benchmark-8k-180-deep", + original_filename="synthetic-8k-180", + analysis_mode="deep", + skip_ai=True, + parallel_workers=4, + dataframe=frame, + write_artifacts=False, + ) + elapsed = time.perf_counter() - started + + payload = { + "benchmark_schema_version": 1, + "workload": { + "rows": int(args.rows), + "columns": int(args.columns), + "kind": "deterministic_wide_numeric_deep", + "seed": 42, + "backend": os.environ.get("FRAMEVITALS_BACKEND", "auto"), + }, + "elapsed_seconds": round(float(elapsed), 6), + "historical_original_seconds": HISTORICAL_ORIGINAL_SECONDS, + "ten_x_target_seconds": round(TEN_X_TARGET_SECONDS, 6), + "speedup_vs_historical_original": round( + HISTORICAL_ORIGINAL_SECONDS / max(elapsed, 1.0e-12), + 4, + ), + "ten_x_target_met": bool(elapsed <= TEN_X_TARGET_SECONDS), + "pipeline_timings_ms": result.get("timings_ms", {}), + "deep_execution": ( + result.get("deep_statistics_v2", {}).get("execution", {}) + if isinstance(result.get("deep_statistics_v2"), dict) + else {} + ), + } + + encoded = json.dumps(payload, indent=2, sort_keys=True) + print(encoded) + if args.output is not None: + args.output.write_text(encoded + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/benchmark_extreme_streaming.py b/benchmarks/benchmark_extreme_streaming.py new file mode 100644 index 0000000..1557575 --- /dev/null +++ b/benchmarks/benchmark_extreme_streaming.py @@ -0,0 +1,169 @@ +"""Virtual ultra-wide streaming benchmark. + +A dense materialization of the target shapes is intentionally avoided. The +benchmark models a projection-capable streaming source and fails if FrameVitals +ever requests the complete ultra-wide schema during analysis. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import time +from types import SimpleNamespace + +import numpy as np +import pyarrow as pa + +import framevitals as fv +from framevitals.sources import DatasetMetadata + + +DEFAULT_ROWS = 1_000_000 +DEFAULT_COLUMNS = 100_000 +VALID_MODES = ("quick", "standard", "deep", "research") + + +class VirtualSchema: + def __init__(self, columns: int): + self.columns = int(columns) + self.dtype = pa.float64() + + def __iter__(self): + for index in range(self.columns): + yield SimpleNamespace(name=f"n{index:05d}", type=self.dtype) + + def field(self, name: str): + index = int(name[1:]) + if index < 0 or index >= self.columns: + raise KeyError(name) + return SimpleNamespace(name=name, type=self.dtype) + + +class VirtualExtremeSource: + def __init__(self, rows: int, columns: int): + self.rows = int(rows) + self.columns = int(columns) + self.schema_view = VirtualSchema(columns) + self.max_requested_columns = 0 + self.unbounded_requests = 0 + self.batches_yielded = 0 + self.rows_yielded = 0 + + def inspect(self): + return DatasetMetadata( + name=f"virtual-{self.rows}-x-{self.columns}", + kind="virtual", + format="synthetic", + rows=self.rows, + columns=self.columns, + size_bytes=self.rows * self.columns * 8, + materialized=False, + supports_projection=True, + supports_streaming=True, + ) + + def schema(self): + return self.schema_view + + def iter_batches(self, *, batch_size=65_536, columns=None): + if columns is None: + self.unbounded_requests += 1 + raise RuntimeError( + f"FrameVitals attempted an unbounded {self.columns:,}-column scan." + ) + + names = list(columns) + self.max_requested_columns = max(self.max_requested_columns, len(names)) + offset = 0 + while offset < self.rows: + take = min(int(batch_size), self.rows - offset) + base = np.arange(offset, offset + take, dtype=np.float64) + values = pa.array((base % 10_007) / 10_007.0) + batch = pa.RecordBatch.from_arrays([values] * len(names), names=names) + self.batches_yielded += 1 + self.rows_yielded += take + yield batch + offset += take + + def load(self): + raise RuntimeError("Virtual extreme source must never materialize fully.") + + def reset_observations(self) -> None: + self.max_requested_columns = 0 + self.unbounded_requests = 0 + self.batches_yielded = 0 + self.rows_yielded = 0 + + +def run_benchmark(*, rows: int, columns: int, mode: str) -> dict: + source = VirtualExtremeSource(rows, columns) + + plan_started = time.perf_counter() + plan = fv.plan(source, mode=mode, workers=4) + plan_seconds = time.perf_counter() - plan_started + + source.reset_observations() + + analysis_started = time.perf_counter() + result = fv.analyze( + source, + mode=mode, + artifacts=False, + workers=4, + ) + analysis_seconds = time.perf_counter() - analysis_started + + streaming = result.get("execution", {}).get("streaming", {}) + working_sample_rows = int(streaming.get("working_sample_rows", 0) or 0) + return { + "benchmark_schema_version": 2, + "workload": { + "rows": rows, + "columns": columns, + "cells": rows * columns, + "dense_float64_raw_gb": round(rows * columns * 8 / 1_000_000_000, 3), + "mode": mode, + "source": "virtual_projection_capable_stream", + }, + "plan_seconds": round(plan_seconds, 6), + "analysis_seconds": round(analysis_seconds, 6), + "scale_class": plan.get("execution_budget", {}).get("scale_class"), + "planning": plan.get("planning_data", {}), + "streaming_execution": streaming, + "pipeline_timings_ms": result.get("timings_ms", {}), + "source_observed": { + "max_requested_columns": source.max_requested_columns, + "unbounded_requests": source.unbounded_requests, + "batches_yielded": source.batches_yielded, + "rows_yielded": source.rows_yielded, + }, + "safety": { + "full_materialization": streaming.get("full_materialization"), + "column_sampled": streaming.get("column_sampled"), + "profiled_columns": streaming.get("profiled_columns"), + "source_columns": streaming.get("source_columns"), + "working_sample_rows": working_sample_rows, + "all_source_rows_scanned": source.rows_yielded == rows, + "unbounded_width_requested": source.unbounded_requests > 0, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--rows", type=int, default=DEFAULT_ROWS) + parser.add_argument("--columns", type=int, default=DEFAULT_COLUMNS) + parser.add_argument("--mode", choices=VALID_MODES, default="deep") + parser.add_argument("--output", type=Path, default=Path("extreme-benchmark-results.json")) + args = parser.parse_args() + + payload = run_benchmark(rows=args.rows, columns=args.columns, mode=args.mode) + encoded = json.dumps(payload, indent=2, sort_keys=True) + args.output.write_text(encoded + "\n", encoding="utf-8") + print(encoded) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/benchmark_profile_scale.py b/benchmarks/benchmark_profile_scale.py new file mode 100644 index 0000000..945bee7 --- /dev/null +++ b/benchmarks/benchmark_profile_scale.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +"""Reproducible FrameVitals scale benchmark. + +The harness runs each scenario in a fresh subprocess so process peak RSS values +are comparable and one scenario's allocator state cannot contaminate another. +It intentionally records measurements rather than enforcing fragile absolute +performance thresholds in CI. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import platform +import resource +import subprocess +import sys +import tempfile +import time +from typing import Any + +import numpy as np +import pandas as pd + + +DEFAULT_ROWS = 100_000 +DEFAULT_NUMERIC_COLUMNS = 100 +DEFAULT_CATEGORICAL_COLUMNS = 5 +DEFAULT_SEED = 42 +SCENARIOS = ("numpy", "auto", "parquet") + + +def _peak_rss_mb() -> float: + raw = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + # Linux reports KiB; macOS reports bytes. + if platform.system() == "Darwin": + return raw / (1024 * 1024) + return raw / 1024 + + +def _frame( + rows: int, + numeric_columns: int, + categorical_columns: int, + seed: int, +) -> pd.DataFrame: + rng = np.random.default_rng(seed) + values = rng.standard_normal((rows, numeric_columns), dtype=np.float64) + if numeric_columns: + values[::97, ::7] = np.nan + frame = pd.DataFrame( + values, + columns=[f"metric_{index:03d}" for index in range(numeric_columns)], + copy=False, + ) + for index in range(categorical_columns): + cardinality = 7 + index * 3 + frame[f"category_{index:02d}"] = np.asarray( + [f"g{index}_{row % cardinality}" for row in range(rows)], + dtype=object, + ) + return frame + + +def _write_parquet_streaming( + path: Path, + *, + rows: int, + numeric_columns: int, + categorical_columns: int, + seed: int, + chunk_rows: int = 20_000, +) -> None: + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError as exc: + raise RuntimeError( + 'Parquet benchmark requires: pip install "framevitals[arrow]"' + ) from exc + + writer = None + try: + offset = 0 + chunk_index = 0 + while offset < rows: + current = min(chunk_rows, rows - offset) + frame = _frame( + current, + numeric_columns, + categorical_columns, + seed + chunk_index, + ) + table = pa.Table.from_pandas(frame, preserve_index=False) + if writer is None: + writer = pq.ParquetWriter(path, table.schema) + writer.write_table(table, row_group_size=min(current, 10_000)) + offset += current + chunk_index += 1 + finally: + if writer is not None: + writer.close() + + +def _result_metadata(result: dict[str, Any]) -> dict[str, Any]: + return { + "shape": result.get("shape"), + "numeric_summary_metadata": result.get("numeric_summary_metadata"), + "correlation_metadata": result.get("correlation_metadata"), + "duplicate_metadata": result.get("duplicate_metadata"), + "streaming_metadata": result.get("streaming_metadata"), + "source_metadata": result.get("source_metadata"), + } + + +def _worker(args: argparse.Namespace) -> int: + import framevitals + from framevitals.backends import backend_status + + started = time.perf_counter() + source_memory_mb = None + + if args.scenario in {"numpy", "auto"}: + if args.scenario == "numpy": + os.environ["FRAMEVITALS_BACKEND"] = "numpy" + else: + os.environ.pop("FRAMEVITALS_BACKEND", None) + frame = _frame( + args.rows, + args.numeric_columns, + args.categorical_columns, + args.seed, + ) + source_memory_mb = round( + float(frame.memory_usage(index=True, deep=True).sum()) / (1024 * 1024), + 3, + ) + result = framevitals.profile(frame) + elif args.scenario == "parquet": + if not args.parquet_path: + raise ValueError("parquet worker requires --parquet-path") + os.environ.pop("FRAMEVITALS_BACKEND", None) + result = framevitals.profile(Path(args.parquet_path)) + else: + raise ValueError(f"Unknown benchmark scenario: {args.scenario}") + + elapsed = time.perf_counter() - started + payload = { + "scenario": args.scenario, + "rows": args.rows, + "numeric_columns": args.numeric_columns, + "categorical_columns": args.categorical_columns, + "total_columns": args.numeric_columns + args.categorical_columns, + "seed": args.seed, + "elapsed_seconds": round(elapsed, 6), + "peak_rss_mb": round(_peak_rss_mb(), 3), + "source_memory_mb": source_memory_mb, + "backend_status": backend_status(), + "result": _result_metadata(result), + "python": platform.python_version(), + "platform": platform.platform(), + } + print(json.dumps(payload, sort_keys=True)) + return 0 + + +def _run_scenario( + script: Path, + scenario: str, + *, + rows: int, + numeric_columns: int, + categorical_columns: int, + seed: int, + parquet_path: Path | None, +) -> dict[str, Any]: + command = [ + sys.executable, + str(script), + "--worker", + "--scenario", + scenario, + "--rows", + str(rows), + "--numeric-columns", + str(numeric_columns), + "--categorical-columns", + str(categorical_columns), + "--seed", + str(seed), + ] + if parquet_path is not None: + command.extend(["--parquet-path", str(parquet_path)]) + + completed = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + env=os.environ.copy(), + ) + lines = [line for line in completed.stdout.splitlines() if line.strip()] + if not lines: + raise RuntimeError(f"Benchmark worker {scenario} produced no JSON output.") + return json.loads(lines[-1]) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=DEFAULT_ROWS) + parser.add_argument("--numeric-columns", type=int, default=DEFAULT_NUMERIC_COLUMNS) + parser.add_argument( + "--categorical-columns", + type=int, + default=DEFAULT_CATEGORICAL_COLUMNS, + ) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument( + "--scenarios", + nargs="+", + choices=SCENARIOS, + default=list(SCENARIOS), + ) + parser.add_argument("--output", type=Path, default=None) + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--scenario", choices=SCENARIOS, default=None, help=argparse.SUPPRESS) + parser.add_argument("--parquet-path", default=None, help=argparse.SUPPRESS) + args = parser.parse_args() + if args.rows < 1: + parser.error("--rows must be at least 1") + if args.numeric_columns < 0 or args.categorical_columns < 0: + parser.error("column counts must be non-negative") + if args.numeric_columns + args.categorical_columns < 1: + parser.error("at least one column is required") + if args.worker and args.scenario is None: + parser.error("--worker requires --scenario") + return args + + +def main() -> int: + args = _parse_args() + if args.worker: + return _worker(args) + + script = Path(__file__).resolve() + results: list[dict[str, Any]] = [] + with tempfile.TemporaryDirectory(prefix="framevitals-benchmark-") as directory: + parquet_path: Path | None = None + if "parquet" in args.scenarios: + parquet_path = Path(directory) / "scale.parquet" + prepare_started = time.perf_counter() + _write_parquet_streaming( + parquet_path, + rows=args.rows, + numeric_columns=args.numeric_columns, + categorical_columns=args.categorical_columns, + seed=args.seed, + ) + parquet_prepare_seconds = round(time.perf_counter() - prepare_started, 6) + else: + parquet_prepare_seconds = None + + for scenario in args.scenarios: + results.append( + _run_scenario( + script, + scenario, + rows=args.rows, + numeric_columns=args.numeric_columns, + categorical_columns=args.categorical_columns, + seed=args.seed, + parquet_path=parquet_path if scenario == "parquet" else None, + ) + ) + + payload = { + "benchmark_schema_version": 1, + "workload": { + "rows": args.rows, + "numeric_columns": args.numeric_columns, + "categorical_columns": args.categorical_columns, + "total_columns": args.numeric_columns + args.categorical_columns, + "seed": args.seed, + }, + "parquet_prepare_seconds": parquet_prepare_seconds, + "measurements": results, + "notes": [ + "Each measurement runs in a fresh process.", + "Peak RSS includes imports and in-memory source construction for DataFrame scenarios.", + "Parquet file preparation occurs outside the measured Parquet worker.", + "No absolute timing threshold is enforced; compare like-for-like machines and commits.", + ], + } + serialized = json.dumps(payload, indent=2, sort_keys=True) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(serialized + "\n", encoding="utf-8") + print(serialized) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/benchmark_real_wide_parquet.py b/benchmarks/benchmark_real_wide_parquet.py new file mode 100644 index 0000000..f0f8880 --- /dev/null +++ b/benchmarks/benchmark_real_wide_parquet.py @@ -0,0 +1,295 @@ +"""Generate and benchmark a real 100,000 x 10,000 Parquet dataset. + +The dataset is written in bounded row groups, so generation never materializes +all one billion cells in RAM. The on-disk Parquet file is then analyzed through +the public ``framevitals.analyze`` API. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import time +from typing import Any + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +import framevitals as fv + + +DEFAULT_ROWS = 100_000 +DEFAULT_COLUMNS = 10_000 +PATTERN_COUNT = 32 +VALID_MODES = ("quick", "standard", "deep", "research") + + +def _pattern_numpy(pattern_id: int, row_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray | None]: + """Return one deterministic int16 pattern and optional null mask.""" + pattern_id = int(pattern_id) % PATTERN_COUNT + if pattern_id == 0: + values = row_ids % 101 + elif pattern_id == 1: + values = 2 * (row_ids % 101) + elif pattern_id == 2: + values = (row_ids * 7 + 3) % 211 + elif pattern_id == 3: + values = np.full(row_ids.shape, 7, dtype=np.int64) + elif pattern_id == 4: + values = (row_ids * 13) % 503 + values = values.copy() + values[row_ids % 997 == 0] = 30_000 + elif pattern_id == 5: + values = (row_ids * 5) % 307 + return values.astype(np.int16, copy=False), (row_ids % 10 == 0) + elif pattern_id == 6: + values = row_ids % 2 + elif pattern_id == 7: + values = np.where(row_ids % 100 < 50, 10, 200) + else: + multiplier = pattern_id * 2 + 1 + values = (row_ids * multiplier + pattern_id * 17) % 997 + return values.astype(np.int16, copy=False), None + + +def _pattern_arrow(pattern_id: int, row_ids: np.ndarray) -> pa.Array: + values, mask = _pattern_numpy(pattern_id, row_ids) + return pa.array(values, mask=mask, type=pa.int16()) + + +def generate_dataset( + path: Path, + *, + rows: int = DEFAULT_ROWS, + columns: int = DEFAULT_COLUMNS, + row_group_rows: int = 10_000, +) -> dict[str, Any]: + """Write the full logical dataset to Parquet using bounded-memory row groups.""" + path.parent.mkdir(parents=True, exist_ok=True) + names = [f"n{index:05d}" for index in range(columns)] + schema = pa.schema([pa.field(name, pa.int16()) for name in names]) + + started = time.perf_counter() + writer = pq.ParquetWriter( + path, + schema=schema, + compression="snappy", + use_dictionary=True, + write_statistics=False, + ) + try: + for start in range(0, rows, row_group_rows): + stop = min(rows, start + row_group_rows) + row_ids = np.arange(start, stop, dtype=np.int64) + patterns = [_pattern_arrow(index, row_ids) for index in range(PATTERN_COUNT)] + arrays = [patterns[index % PATTERN_COUNT] for index in range(columns)] + batch = pa.RecordBatch.from_arrays(arrays, schema=schema) + writer.write_table( + pa.Table.from_batches([batch], schema=schema), + row_group_size=len(row_ids), + ) + finally: + writer.close() + + elapsed = time.perf_counter() - started + parquet = pq.ParquetFile(path) + return { + "path": str(path), + "rows": int(parquet.metadata.num_rows), + "columns": int(parquet.metadata.num_columns), + "cells": int(parquet.metadata.num_rows) * int(parquet.metadata.num_columns), + "file_size_mb": round(path.stat().st_size / 1_000_000, 3), + "dense_int16_raw_gb": round(rows * columns * 2 / 1_000_000_000, 3), + "generation_seconds": round(elapsed, 6), + "row_groups": int(parquet.metadata.num_row_groups), + "pattern_count": PATTERN_COUNT, + } + + +def _expected_metrics(rows: int) -> dict[int, dict[str, Any]]: + row_ids = np.arange(rows, dtype=np.int64) + expected: dict[int, dict[str, Any]] = {} + for pattern_id in range(PATTERN_COUNT): + values, mask = _pattern_numpy(pattern_id, row_ids) + valid = values if mask is None else values[~mask] + expected[pattern_id] = { + "missing": 0 if mask is None else int(mask.sum()), + "count": int(valid.size), + "mean": float(valid.astype(np.float64).mean()) if valid.size else None, + "min": int(valid.min()) if valid.size else None, + "max": int(valid.max()) if valid.size else None, + } + return expected + + +def _accuracy_checks(result: dict[str, Any], *, rows: int) -> dict[str, Any]: + profile = result.get("profile", {}) + columns = list(profile.get("columns", [])) + missing_counts = profile.get("missing_counts", {}) + numeric_summary = profile.get("numeric_summary", {}) + expected = _expected_metrics(rows) + + missing_matches = 0 + count_matches = 0 + minmax_matches = 0 + mean_errors: list[float] = [] + expected_missing_total = 0 + + for column in columns: + index = int(str(column)[1:]) + truth = expected[index % PATTERN_COUNT] + expected_missing_total += int(truth["missing"]) + if int(missing_counts.get(column, -1)) == int(truth["missing"]): + missing_matches += 1 + + summary = numeric_summary.get(column, {}) + if int(summary.get("count", -1)) == int(truth["count"]): + count_matches += 1 + if summary.get("min") == truth["min"] and summary.get("max") == truth["max"]: + minmax_matches += 1 + observed_mean = summary.get("mean") + if observed_mean is not None and truth["mean"] is not None: + mean_errors.append(abs(float(observed_mean) - float(truth["mean"]))) + + expected_health_missing = expected_missing_total / max(rows * len(columns), 1) * 100.0 + observed_health_missing = float( + result.get("health", {}).get("details", {}).get("missing_percent", 0.0) or 0.0 + ) + + deep_stats = result.get("deep_statistics_v2", {}) + numeric_stats = deep_stats.get("numeric_statistics", {}) if isinstance(deep_stats, dict) else {} + deep_mean_errors: list[float] = [] + exact_once_columns = 0 + for column, summary in numeric_stats.items(): + if not isinstance(summary, dict) or summary.get("mean") is None: + continue + index = int(str(column)[1:]) + truth_mean = expected[index % PATTERN_COUNT]["mean"] + if truth_mean is not None: + deep_mean_errors.append(abs(float(summary["mean"]) - float(truth_mean))) + provenance = summary.get("summary_provenance", {}) + if isinstance(provenance, dict) and provenance.get("scope") == "full_stream": + exact_once_columns += 1 + + timings = result.get("timings_ms", {}) + phase3 = timings.get("phase3_tasks", {}) if isinstance(timings, dict) else {} + return { + "profiled_columns_checked": len(columns), + "missing_count_exact_matches": missing_matches, + "numeric_count_exact_matches": count_matches, + "numeric_minmax_exact_matches": minmax_matches, + "profile_mean_max_abs_error": round(max(mean_errors), 6) if mean_errors else None, + "profile_mean_mean_abs_error": round(float(np.mean(mean_errors)), 6) if mean_errors else None, + "expected_projected_missing_percent": round(expected_health_missing, 6), + "health_missing_percent": round(observed_health_missing, 6), + "health_missing_abs_error": round(abs(observed_health_missing - expected_health_missing), 6), + "deep_numeric_columns_checked": len(numeric_stats), + "deep_exact_once_columns": exact_once_columns, + "deep_mean_max_abs_error": round(max(deep_mean_errors), 6) if deep_mean_errors else None, + "deep_mean_mean_abs_error": round(float(np.mean(deep_mean_errors)), 6) if deep_mean_errors else None, + "phase3_tasks_present": sorted(phase3) if isinstance(phase3, dict) else [], + "full_pipeline_sections_present": sorted( + key + for key in ( + "profile", + "column_roles", + "health", + "ml_readiness", + "quality_diagnostics", + "advanced", + "deep_statistics_v2", + "anomalies_v2", + "time_series", + ) + if key in result + ), + } + + +def benchmark_dataset(path: Path, *, mode: str) -> dict[str, Any]: + parquet = pq.ParquetFile(path) + rows = int(parquet.metadata.num_rows) + columns = int(parquet.metadata.num_columns) + + started = time.perf_counter() + result = fv.analyze(path, mode=mode, artifacts=False, workers=4) + analysis_seconds = time.perf_counter() - started + + streaming = result.get("execution", {}).get("streaming", {}) + profile = result.get("profile", {}) + numeric_metadata = profile.get("numeric_summary_metadata", {}) + streaming_metadata = profile.get("streaming_metadata", {}) + return { + "benchmark_schema_version": 2, + "workload": { + "rows": rows, + "columns": columns, + "cells": rows * columns, + "file_size_mb": round(path.stat().st_size / 1_000_000, 3), + "dense_int16_raw_gb": round(rows * columns * 2 / 1_000_000_000, 3), + "mode": mode, + "source": "real_parquet_file", + }, + "backend": { + "numeric_backend": streaming_metadata.get("numeric_backend"), + "numeric_method": ( + numeric_metadata.get("method") + if isinstance(numeric_metadata, dict) + else None + ), + }, + "analysis_seconds": round(analysis_seconds, 6), + "pipeline_timings_ms": result.get("timings_ms", {}), + "streaming_execution": streaming, + "accuracy": _accuracy_checks(result, rows=rows), + "safety": { + "full_materialization": streaming.get("full_materialization"), + "source_rows": streaming.get("source_rows"), + "source_columns": streaming.get("source_columns"), + "profiled_columns": streaming.get("profiled_columns"), + "working_sample_rows": streaming.get("working_sample_rows"), + "column_sampled": streaming.get("column_sampled"), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=Path, default=Path("wide-100k-x-10k.parquet")) + parser.add_argument("--rows", type=int, default=DEFAULT_ROWS) + parser.add_argument("--columns", type=int, default=DEFAULT_COLUMNS) + parser.add_argument("--row-group-rows", type=int, default=10_000) + parser.add_argument("--mode", choices=VALID_MODES) + parser.add_argument("--generate-only", action="store_true") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + if args.generate_only: + payload = generate_dataset( + args.dataset, + rows=args.rows, + columns=args.columns, + row_group_rows=args.row_group_rows, + ) + else: + if not args.dataset.exists(): + generate_dataset( + args.dataset, + rows=args.rows, + columns=args.columns, + row_group_rows=args.row_group_rows, + ) + if args.mode is None: + parser.error("--mode is required unless --generate-only is used") + payload = benchmark_dataset(args.dataset, mode=args.mode) + + encoded = json.dumps(payload, indent=2, sort_keys=True) + if args.output is not None: + args.output.write_text(encoded + "\n", encoding="utf-8") + print(encoded) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/performance_budgets.json b/benchmarks/performance_budgets.json new file mode 100644 index 0000000..505bb40 --- /dev/null +++ b/benchmarks/performance_budgets.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1", + "notes": "Initial ceilings are intentionally generous. Tighten only after collecting stable hosted-run history.", + "scenarios": { + "core_profile_150k_x12": { + "max_seconds": 25.0, + "max_peak_rss_mb": 1200.0 + }, + "parquet_profile_150k_x12": { + "max_seconds": 30.0, + "max_peak_rss_mb": 1400.0 + } + } +} diff --git a/benchmarks/performance_guardrail.py b/benchmarks/performance_guardrail.py new file mode 100644 index 0000000..e81d5ed --- /dev/null +++ b/benchmarks/performance_guardrail.py @@ -0,0 +1,198 @@ +"""Deterministic catastrophic-regression guardrails for FrameVitals profiling. + +These checks are deliberately broader than microbenchmarks. Hosted CI hardware is +noisy, so the initial budgets only fail large time/RSS regressions. Raw measurements +are written as JSON so the ceilings can be tightened later from observed history. +""" + +from __future__ import annotations + +import argparse +import json +import os +import resource +import tempfile +import time +from pathlib import Path +from typing import Any, Callable + +import numpy as np +import pandas as pd + +import framevitals as fv + + +ROWS = 150_000 +NUMERIC_COLUMNS = 8 +CATEGORICAL_COLUMNS = 4 + + +def _peak_rss_mb() -> float: + usage = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + # Linux reports KiB; macOS reports bytes. CI is Linux, but keep the script + # useful locally as well. + if os.uname().sysname.lower() == "darwin": + return usage / (1024.0 * 1024.0) + return usage / 1024.0 + + +def _frame(rows: int = ROWS) -> pd.DataFrame: + rng = np.random.default_rng(42) + payload: dict[str, Any] = {} + for index in range(NUMERIC_COLUMNS): + values = rng.normal(loc=index, scale=1.0 + index / 10.0, size=rows) + values[::997] = np.nan + payload[f"value_{index}"] = values + for index in range(CATEGORICAL_COLUMNS): + payload[f"group_{index}"] = np.take( + np.array([f"g{index}-{item}" for item in range(12)], dtype=object), + np.arange(rows) % 12, + ) + return pd.DataFrame(payload) + + +def _measure(name: str, operation: Callable[[], dict[str, Any]]) -> dict[str, Any]: + before_rss = _peak_rss_mb() + started = time.perf_counter() + result = operation() + elapsed = time.perf_counter() - started + after_rss = _peak_rss_mb() + return { + "name": name, + "elapsed_seconds": round(float(elapsed), 6), + "peak_rss_mb": round(float(after_rss), 3), + "rss_growth_mb": round(max(0.0, float(after_rss - before_rss)), 3), + "result": result, + } + + +def _core_profile() -> dict[str, Any]: + frame = _frame() + profile = fv.profile(frame) + assert profile["shape"] == {"rows": ROWS, "columns": NUMERIC_COLUMNS + CATEGORICAL_COLUMNS} + return { + "rows": ROWS, + "columns": NUMERIC_COLUMNS + CATEGORICAL_COLUMNS, + "diagnostic": profile.diagnostic, + } + + +def _parquet_profile() -> dict[str, Any]: + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError as exc: # pragma: no cover - workflow installs arrow + raise RuntimeError( + 'Parquet guardrail requires: pip install "framevitals[arrow]"' + ) from exc + + frame = _frame() + with tempfile.TemporaryDirectory(prefix="framevitals-perf-") as directory: + path = Path(directory) / "guardrail.parquet" + pq.write_table( + pa.Table.from_pandas(frame, preserve_index=False), + path, + row_group_size=8_192, + ) + del frame + profile = fv.profile(path) + + assert profile["shape"] == {"rows": ROWS, "columns": NUMERIC_COLUMNS + CATEGORICAL_COLUMNS} + streaming = profile.get("streaming_metadata", {}) + assert streaming.get("enabled") is True + assert streaming.get("full_materialization") is False + return { + "rows": ROWS, + "columns": NUMERIC_COLUMNS + CATEGORICAL_COLUMNS, + "diagnostic": profile.diagnostic, + "streaming": True, + } + + +SCENARIOS: dict[str, Callable[[], dict[str, Any]]] = { + "core_profile_150k_x12": _core_profile, + "parquet_profile_150k_x12": _parquet_profile, +} + + +def _load_budgets(path: Path) -> dict[str, dict[str, float]]: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schema_version") != "1": + raise ValueError("Unsupported performance budget schema version.") + scenarios = payload.get("scenarios") + if not isinstance(scenarios, dict): + raise ValueError("Performance budget file is missing scenarios.") + return scenarios + + +def run_guardrails(budget_path: Path) -> dict[str, Any]: + budgets = _load_budgets(budget_path) + measurements: list[dict[str, Any]] = [] + failures: list[dict[str, Any]] = [] + + for name, operation in SCENARIOS.items(): + if name not in budgets: + raise ValueError(f"Missing performance budget for scenario: {name}") + budget = budgets[name] + measurement = _measure(name, operation) + max_seconds = float(budget["max_seconds"]) + max_peak_rss_mb = float(budget["max_peak_rss_mb"]) + measurement["budget"] = { + "max_seconds": max_seconds, + "max_peak_rss_mb": max_peak_rss_mb, + } + measurement["passed"] = ( + measurement["elapsed_seconds"] <= max_seconds + and measurement["peak_rss_mb"] <= max_peak_rss_mb + ) + measurements.append(measurement) + + if measurement["elapsed_seconds"] > max_seconds: + failures.append({ + "scenario": name, + "metric": "elapsed_seconds", + "observed": measurement["elapsed_seconds"], + "budget": max_seconds, + }) + if measurement["peak_rss_mb"] > max_peak_rss_mb: + failures.append({ + "scenario": name, + "metric": "peak_rss_mb", + "observed": measurement["peak_rss_mb"], + "budget": max_peak_rss_mb, + }) + + return { + "schema_version": "1", + "rows": ROWS, + "scenario_count": len(measurements), + "passed": not failures, + "measurements": measurements, + "failures": failures, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--budgets", + type=Path, + default=Path(__file__).with_name("performance_budgets.json"), + ) + parser.add_argument("--output", type=Path, default=None) + return parser + + +def main() -> int: + args = _parser().parse_args() + result = run_guardrails(args.budgets) + rendered = json.dumps(result, indent=2, default=str) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n", encoding="utf-8") + print(rendered) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json b/benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json new file mode 100644 index 0000000..3640920 --- /dev/null +++ b/benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json @@ -0,0 +1,322 @@ +{ + "benchmark_schema_version": 1, + "comparison": "FrameVitals 0.2.0 vs 0.1.0", + "old_ref": "v0.1.0@3da1432168fbfcb3dbe99fcfb6f6200f5e63214b", + "new_ref": "develop/august@05b11e594995a3833ec08f5b4a8a145197bf4cab", + "dataset": { + "rows": 10000, + "columns": 64, + "cells": 640000, + "format": "csv", + "bytes": 2811188, + "c000": { + "count": 10000, + "mean": -1.4965, + "min": -1000, + "max": 998 + } + }, + "methodology": { + "runner": "GitHub Actions ubuntu-latest", + "python": "3.11", + "warmups_per_version_mode": 1, + "measured_repetitions_per_version_mode": 3, + "measurement_order": "ABBAAB", + "public_api": "framevitals.analyze(path, mode=..., artifacts=False)", + "scope": "all 10,000 rows and all 64 columns in both releases", + "dependency_stack": { + "numpy": "1.26.0", + "pandas": "2.2.0", + "scipy": "1.13.0", + "statsmodels": "0.14.0", + "scikit-learn": "1.5.0", + "pyarrow": "25.0.0" + } + }, + "modes": { + "quick": { + "0.1.0": { + "wall_seconds": [ + 1.6478449049999995, + 1.619613931999993, + 1.6437554540000008 + ], + "median_wall_seconds": 1.6437554540000008, + "min_wall_seconds": 1.619613931999993, + "max_wall_seconds": 1.6478449049999995, + "peak_rss_mb": [ + 234.6953125, + 234.5390625, + 234.6796875 + ], + "median_peak_rss_mb": 234.6796875, + "backend": { + "selected": "legacy-python", + "native_available": false + }, + "profiled_columns": 64, + "full_materialization": null + }, + "0.2.0": { + "wall_seconds": [ + 0.6861838410000018, + 0.6908588409999936, + 0.691870496000007 + ], + "median_wall_seconds": 0.6908588409999936, + "min_wall_seconds": 0.6861838410000018, + "max_wall_seconds": 0.691870496000007, + "peak_rss_mb": [ + 239.72265625, + 238.875, + 239.515625 + ], + "median_peak_rss_mb": 239.515625, + "backend": { + "selected": "rust", + "native_available": true, + "environment_override": "rust", + "eligible": [ + "numpy", + "rust" + ] + }, + "profiled_columns": 64, + "full_materialization": null + }, + "speedup_x": 2.3792927823297783, + "wall_time_reduction_percent": 57.970704260246166, + "peak_rss_reduction_percent": -2.0606544825060746 + }, + "standard": { + "0.1.0": { + "wall_seconds": [ + 141.083503924, + 141.13010660999998, + 140.708747796 + ], + "median_wall_seconds": 141.083503924, + "min_wall_seconds": 140.708747796, + "max_wall_seconds": 141.13010660999998, + "peak_rss_mb": [ + 2737.171875, + 2736.421875, + 2736.7265625 + ], + "median_peak_rss_mb": 2736.7265625, + "backend": { + "selected": "legacy-python", + "native_available": false + }, + "profiled_columns": 64, + "full_materialization": null + }, + "0.2.0": { + "wall_seconds": [ + 0.8411333409999884, + 0.822897117000025, + 0.8394920780000348 + ], + "median_wall_seconds": 0.8394920780000348, + "min_wall_seconds": 0.822897117000025, + "max_wall_seconds": 0.8411333409999884, + "peak_rss_mb": [ + 265.33984375, + 264.4296875, + 264.8984375 + ], + "median_peak_rss_mb": 264.8984375, + "backend": { + "selected": "rust", + "native_available": true, + "environment_override": "rust", + "eligible": [ + "numpy", + "rust" + ] + }, + "profiled_columns": 64, + "full_materialization": null + }, + "speedup_x": 168.05817186519556, + "wall_time_reduction_percent": 99.40496794121852, + "peak_rss_reduction_percent": 90.32060998969457 + } + }, + "measurements": [ + { + "version": "0.1.0", + "mode": "quick", + "elapsed_seconds": 1.6478449049999995, + "peak_rss_mb": 234.6953125, + "profiled_columns": 64, + "backend_status": { + "selected": "legacy-python", + "native_available": false + }, + "full_materialization": null + }, + { + "version": "0.1.0", + "mode": "quick", + "elapsed_seconds": 1.619613931999993, + "peak_rss_mb": 234.5390625, + "profiled_columns": 64, + "backend_status": { + "selected": "legacy-python", + "native_available": false + }, + "full_materialization": null + }, + { + "version": "0.1.0", + "mode": "quick", + "elapsed_seconds": 1.6437554540000008, + "peak_rss_mb": 234.6796875, + "profiled_columns": 64, + "backend_status": { + "selected": "legacy-python", + "native_available": false + }, + "full_materialization": null + }, + { + "version": "0.2.0", + "mode": "quick", + "elapsed_seconds": 0.6861838410000018, + "peak_rss_mb": 239.72265625, + "profiled_columns": 64, + "backend_status": { + "selected": "rust", + "native_available": true, + "environment_override": "rust", + "eligible": [ + "numpy", + "rust" + ] + }, + "full_materialization": null + }, + { + "version": "0.2.0", + "mode": "quick", + "elapsed_seconds": 0.6908588409999936, + "peak_rss_mb": 238.875, + "profiled_columns": 64, + "backend_status": { + "selected": "rust", + "native_available": true, + "environment_override": "rust", + "eligible": [ + "numpy", + "rust" + ] + }, + "full_materialization": null + }, + { + "version": "0.2.0", + "mode": "quick", + "elapsed_seconds": 0.691870496000007, + "peak_rss_mb": 239.515625, + "profiled_columns": 64, + "backend_status": { + "selected": "rust", + "native_available": true, + "environment_override": "rust", + "eligible": [ + "numpy", + "rust" + ] + }, + "full_materialization": null + }, + { + "version": "0.1.0", + "mode": "standard", + "elapsed_seconds": 141.083503924, + "peak_rss_mb": 2737.171875, + "profiled_columns": 64, + "backend_status": { + "selected": "legacy-python", + "native_available": false + }, + "full_materialization": null + }, + { + "version": "0.1.0", + "mode": "standard", + "elapsed_seconds": 141.13010660999998, + "peak_rss_mb": 2736.421875, + "profiled_columns": 64, + "backend_status": { + "selected": "legacy-python", + "native_available": false + }, + "full_materialization": null + }, + { + "version": "0.1.0", + "mode": "standard", + "elapsed_seconds": 140.708747796, + "peak_rss_mb": 2736.7265625, + "profiled_columns": 64, + "backend_status": { + "selected": "legacy-python", + "native_available": false + }, + "full_materialization": null + }, + { + "version": "0.2.0", + "mode": "standard", + "elapsed_seconds": 0.8411333409999884, + "peak_rss_mb": 265.33984375, + "profiled_columns": 64, + "backend_status": { + "selected": "rust", + "native_available": true, + "environment_override": "rust", + "eligible": [ + "numpy", + "rust" + ] + }, + "full_materialization": null + }, + { + "version": "0.2.0", + "mode": "standard", + "elapsed_seconds": 0.822897117000025, + "peak_rss_mb": 264.4296875, + "profiled_columns": 64, + "backend_status": { + "selected": "rust", + "native_available": true, + "environment_override": "rust", + "eligible": [ + "numpy", + "rust" + ] + }, + "full_materialization": null + }, + { + "version": "0.2.0", + "mode": "standard", + "elapsed_seconds": 0.8394920780000348, + "peak_rss_mb": 264.8984375, + "profiled_columns": 64, + "backend_status": { + "selected": "rust", + "native_available": true, + "environment_override": "rust", + "eligible": [ + "numpy", + "rust" + ] + }, + "full_materialization": null + } + ] +} diff --git a/benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json b/benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json new file mode 100644 index 0000000..c4bd357 --- /dev/null +++ b/benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json @@ -0,0 +1,104 @@ +{ + "benchmark_schema_version": 1, + "comparison": "FrameVitals 0.2.0 vs 0.1.0 same-dataset statistical accuracy", + "dataset": { + "bytes": 2811188, + "cells": 640000, + "columns": 64, + "format": "csv", + "generator": "((row * (col + 3) + col * 17) % 2001 - 1000).astype(int16)", + "rows": 10000, + "same_as_release_performance_run": 32010158292 + }, + "delta_new_minus_old": { + "max_exact_fact_absolute_error": 0.0, + "max_mean_absolute_error": 0.0, + "max_quantile_absolute_error": 4.196000000000026, + "max_shape_absolute_error": 0.0, + "max_std_absolute_error": 0.0, + "pearson_absolute_error": 0.0 + }, + "evidence_runs": { + "accuracy_full_legacy_and_initial_native_run": 32014979365, + "native_shape_corrected_run": 32015665811, + "same_dataset_performance_run": 32010158292 + }, + "methodology": { + "dependency_stack": { + "numpy": "1.26.0", + "pandas": "2.2.0", + "pyarrow": "25.0.0", + "scikit-learn": "1.5.0", + "scipy": "1.13.0", + "statsmodels": "0.14.0" + }, + "notes": [ + "0.1.0 output came from accuracy run 32014979365.", + "0.2.0 shape fields were reprobed in run 32015665811 because the initial extractor looked only in deep_statistics_v2 instead of the profile's full-stream M3/M4 fields.", + "The product engine source under test is develop/august@13e16791; temporary benchmark-only files did not modify FrameVitals engine code." + ], + "public_api": "framevitals.analyze(path, mode='standard', artifacts=False)", + "python": "3.11", + "runner": "GitHub Actions ubuntu-latest", + "scope": "five representative numeric columns plus Pearson correlation; full 10,000 x 64 Standard analysis in both releases", + "truth_source": "serialized CSV recomputed independently with pandas/scipy" + }, + "new_ref": "develop/august@13e16791f6f3e1fee74ade79c68cbeaf60cee9dc", + "old_ref": "v0.1.0@3da1432168fbfcb3dbe99fcfb6f6200f5e63214b", + "quantile_error_context": { + "max_absolute_error_units": 4.196000000000026, + "max_error_percent_of_column_range": 0.21001001001001132, + "mean_absolute_error_units": 1.589200000000014, + "mean_error_percent_of_column_range": 0.07949557557557627, + "native_quantile_relative_accuracy_setting": 0.01, + "tracked_quantile_values": 15 + }, + "tracked_columns": ["c000", "c001", "c007", "c031", "c063"], + "truth_method": "independent pandas/scipy calculation from the serialized CSV", + "versions": { + "0.1.0": { + "backend": {"native_available": false, "selected": "legacy-python"}, + "column_absolute_errors": { + "c000": {"count": 0.0, "kurtosis": 2.565117664943273e-07, "max": 0.0, "mean": 0.0004999999999999449, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 9.411780260499463e-08, "std": 0.0004745087312585383}, + "c001": {"count": 0.0, "kurtosis": 3.374129284861027e-07, "max": 0.0, "mean": 0.00040000000000001146, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 3.7097656380637127e-07, "std": 0.0003787506919934458}, + "c007": {"count": 0.0, "kurtosis": 4.12596387722175e-07, "max": 0.0, "mean": 0.0005000000000000004, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 3.5892954760251804e-07, "std": 0.00033771722223718825}, + "c031": {"count": 0.0, "kurtosis": 4.915813716088735e-07, "max": 0.0, "mean": 0.0005000000000000004, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 2.2877938012566199e-07, "std": 5.640241624860209e-07}, + "c063": {"count": 0.0, "kurtosis": 1.1517126541349398e-07, "max": 0.0, "mean": 0.0, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 0.0, "q75": 0.0, "skewness": 4.42799937819564e-07, "std": 6.505523697342142e-05} + }, + "summary": { + "max_exact_fact_absolute_error": 0.0, + "max_mean_absolute_error": 0.0005000000000000004, + "max_quantile_absolute_error": 0.0, + "max_shape_absolute_error": 4.915813716088735e-07, + "max_std_absolute_error": 0.0004745087312585383, + "mean_mean_absolute_error": 0.00037999999999999146, + "mean_quantile_absolute_error": 0.0, + "mean_shape_absolute_error": 3.1088769516840826e-07, + "pearson_absolute_error": 0.00014689456555781744, + "shape_values_unavailable": 0 + } + }, + "0.2.0": { + "backend": {"eligible": ["numpy", "rust"], "environment_override": "rust", "native_available": true, "selected": "rust"}, + "column_absolute_errors": { + "c000": {"count": 0.0, "kurtosis": 2.565117664943273e-07, "max": 0.0, "mean": 0.0004999999999999449, "median": 0.010000000000000009, "min": 0.0, "missing": 0.0, "q25": 4.196000000000026, "q75": 0.053999999999973625, "skewness": 9.411780260499463e-08, "std": 0.0004745087312585383}, + "c001": {"count": 0.0, "kurtosis": 3.374129284861027e-07, "max": 0.0, "mean": 0.00040000000000001146, "median": 0.0, "min": 0.0, "missing": 0.0, "q25": 2.1960000000000264, "q75": 2.1960000000000264, "skewness": 3.7097656380637127e-07, "std": 0.0003787506919934458}, + "c007": {"count": 0.0, "kurtosis": 4.12596387722175e-07, "max": 0.0, "mean": 0.0005000000000000004, "median": 0.5, "min": 0.0, "missing": 0.0, "q25": 1.4460000000000264, "q75": 2.4460000000000264, "skewness": 3.5892954760251804e-07, "std": 0.00033771722223718825}, + "c031": {"count": 0.0, "kurtosis": 4.915813716088735e-07, "max": 0.0, "mean": 0.0005000000000000004, "median": 0.5, "min": 0.0, "missing": 0.0, "q25": 1.4460000000000264, "q75": 2.4460000000000264, "skewness": 2.2877938012566199e-07, "std": 5.640241624860209e-07}, + "c063": {"count": 0.0, "kurtosis": 1.1517126541349398e-07, "max": 0.0, "mean": 0.0, "median": 0.010000000000000009, "min": 0.0, "missing": 0.0, "q25": 4.196000000000026, "q75": 2.1960000000000264, "skewness": 4.42799937819564e-07, "std": 6.505523697342142e-05} + }, + "summary": { + "max_exact_fact_absolute_error": 0.0, + "max_mean_absolute_error": 0.0005000000000000004, + "max_quantile_absolute_error": 4.196000000000026, + "max_shape_absolute_error": 4.915813716088735e-07, + "max_std_absolute_error": 0.0004745087312585383, + "mean_mean_absolute_error": 0.00037999999999999146, + "mean_quantile_absolute_error": 1.589200000000014, + "mean_shape_absolute_error": 3.1088769516840826e-07, + "pearson_absolute_error": 0.00014689456555781744, + "shape_values_unavailable": 0 + } + } + } +} diff --git a/docs/architecture-proposal/BENCHMARK_ACCEPTANCE_PLAN.md b/docs/architecture-proposal/BENCHMARK_ACCEPTANCE_PLAN.md new file mode 100644 index 0000000..f5c153d --- /dev/null +++ b/docs/architecture-proposal/BENCHMARK_ACCEPTANCE_PLAN.md @@ -0,0 +1,203 @@ +# FrameVitals Benchmark and Acceptance Plan + +A comprehensive analysis library needs explicit gates for performance and correctness. This document proposes how new modules should be evaluated before they enter standard/deep/exhaustive presets. + +## Why this matters + +FrameVitals should not equate “more analysis” with “better analysis.” A new method can be valuable only if it: +- produces useful additional evidence +- behaves correctly on edge cases +- has predictable resource cost +- degrades gracefully +- does not make ordinary analysis disproportionately slower + +## Benchmark dimensions + +Each candidate analysis should be measured across: + +### Correctness +- expected findings on golden datasets +- no false structural assumptions on unsupported data +- stable JSON/schema output +- deterministic results when seeded +- correct handling of missing/infinite/mixed values + +### Runtime +- stage wall time +- percentage of total analysis time +- cold vs warm/cache-aware time where relevant + +### Memory +- peak resident memory where measurable +- temporary dataframe/array copies +- model/runtime memory for optional ML/DL features + +### Scalability +- tall datasets +- wide datasets +- high-cardinality categorical data +- long text columns +- large numeric matrices + +### Reproducibility +- package version +- config hash +- random seed +- backend/runtime +- optional pack versions +- model ID/version/checksum + +## Proposed benchmark dataset families + +Synthetic/golden fixtures should cover at least: + +1. **Clean mixed tabular** — representative numeric + categorical + date columns. +2. **Missingness-heavy** — MCAR-like, patterned, and column-dependent missingness. +3. **Duplicate/key issues** — duplicate rows, duplicate candidate IDs, conflicting records. +4. **High cardinality** — IDs, codes, near-unique categoricals. +5. **Outlier/anomaly** — univariate and multivariate injected anomalies. +6. **Drift numeric** — mean/variance/tail/shape changes. +7. **Drift categorical** — frequency shifts, new/missing categories. +8. **Target classification** — balanced and imbalanced labels, leakage and proxy leakage. +9. **Target regression** — nonlinear and linear signal, leakage cases. +10. **Time series** — regular/irregular timestamps, trend, seasonality, gaps, anomalies. +11. **Text** — short categories, long text, duplicates, semantic shift where model packs are tested. +12. **Adversarial dtype** — numeric-looking strings, mixed dates, booleans, infinities, malformed input. +13. **Wide** — hundreds/thousands of columns with modest rows. +14. **Tall** — large row counts with a small/medium number of columns. + +## Standard size classes + +Use named classes rather than hardcoding expectations around one machine: + +| Class | Example purpose | +|---|---| +| Tiny | unit/smoke behavior | +| Small | interactive notebook/CLI usage | +| Medium | common local analytics workload | +| Large | sampling/streaming/backend strategy validation | +| Wide | column-scaling validation | + +Exact row/column counts should be defined in the benchmark suite and can evolve as performance improves. + +## Per-analysis cost classes + +Every analysis registry entry should declare an expected cost class: + +- `tiny` — metadata/simple cached values +- `low` — one lightweight vectorized pass +- `medium` — correlations/statistical tests/model-like operations +- `high` — ensemble, large pairwise relationships, expensive transforms +- `very_high` — deep models, embeddings, exhaustive pairwise/search procedures + +The planner uses this declaration together with observed dimensions and configuration budgets. + +## Default-preset admission rules + +### Quick +An analysis belongs in `quick` only if it is: +- highly reliable +- broadly applicable +- low cost +- useful without extensive interpretation + +### Standard +An analysis belongs in `standard` when: +- value is high for ordinary datasets +- cost is bounded/predictable +- it has strong fallback behavior + +### Deep +An analysis belongs in `deep` when: +- it adds meaningful evidence beyond standard +- higher cost is justified +- output is still interpretable + +### Exhaustive +An analysis can be available in `exhaustive` when: +- applicability is explicit +- resource estimates exist +- optional requirements are declared +- failures can be isolated + +Being present in the codebase does not automatically qualify an analysis for any preset. + +## Regression policy + +Performance changes should be reviewed at the stage level rather than only using total runtime. + +Flag changes when: +- an existing stage becomes materially slower without an intentional tradeoff +- memory copies increase unexpectedly +- a default preset begins running a previously optional expensive module +- cache/context reuse regresses + +Do not enforce overly tight microbenchmark thresholds in noisy CI environments. Prefer broad regression bands, repeated samples, and trend reporting. + +## Model acceptance rules + +An optional ML/DL diagnostic should have a stronger acceptance bar. + +Before inclusion: +1. define exactly which analytical question it answers +2. compare against simpler deterministic/classical baselines +3. measure incremental detection/value, not only model metrics +4. measure CPU runtime and memory +5. document GPU behavior if supported +6. test deterministic seeds where possible +7. document failure/fallback behavior +8. verify model/runtime license compatibility +9. record model/version metadata in results + +A deep model should not be enabled in the standard preset simply because it performs slightly better on one benchmark. + +## Report-quality tests + +Generated output is part of correctness. + +Test: +- JSON serialization +- HTML contains required sections +- no missing/invalid links/assets for self-contained reports +- terminal renderer handles narrow/non-color terminals +- privacy mode does not expose raw sensitive examples +- report generation does not re-run expensive analysis + +## Backend parity + +As Polars/PyArrow/Narwhals support grows, parity tests should focus on **semantic equivalence**, not byte-for-byte output. + +Examples: +- same column semantic roles +- same missing/duplicate counts +- same contract verdicts +- same drift severity class within numerical tolerance +- same findings where algorithms are equivalent + +Backend-specific optimized approximations should clearly record their method. + +## Suggested benchmark command surface + +Future developer tooling could expose: + +```bash +python -m benchmarks.run --suite core +python -m benchmarks.run --suite drift +python -m benchmarks.run --suite large +python -m benchmarks.compare baseline.json candidate.json +``` + +A user-facing `framevitals benchmark` command is not necessary initially; these are primarily development/release tools. + +## Release gate summary + +Before a significant analysis capability moves into a default preset: +- correctness fixtures pass +- serialization/report tests pass +- runtime/memory impact is understood +- fallback behavior is tested +- applicability is registered +- documentation explains the result +- no silent install/download behavior exists + +This keeps “complete and exhaustive” compatible with “fast and trustworthy.” diff --git a/docs/architecture-proposal/CAPABILITY_PACKS.md b/docs/architecture-proposal/CAPABILITY_PACKS.md new file mode 100644 index 0000000..820d2d9 --- /dev/null +++ b/docs/architecture-proposal/CAPABILITY_PACKS.md @@ -0,0 +1,328 @@ +# FrameVitals Capability Pack Specification + +This document defines how optional capabilities can grow without turning the base install into a huge dependency bundle. + +## Design goals + +- The base `framevitals` install must remain useful by itself. +- Optional packs should add clearly defined capabilities. +- Installation and enablement are separate states. +- Model weights should not be bundled into PyPI wheels unless tiny and justified. +- Deep-learning frameworks must stay optional. +- Every optional capability must fail gracefully and preserve deterministic fallbacks when possible. + +## Proposed packs + +### `core` +Installed by default. + +Scope: +- CSV/TSV/basic tabular loading +- structural profiling +- data quality +- health score +- core statistics +- contracts/validation +- drift basics +- result schema +- standard CLI + +Goal: a strong data-analysis/data-health library even with no extras. + +--- + +### `viz` +Purpose: rendering and rich human-facing reports. + +Potential contents: +- plotting dependencies +- standalone HTML report helpers +- PDF/chart rendering +- notebook visual components + +CLI/TUI label: **Visual Reports** + +--- + +### `excel` +Purpose: XLS/XLSX ingestion. + +Potential contents: +- `openpyxl` +- `xlrd` where still required + +CLI/TUI label: **Excel Support** + +--- + +### `ml` +Purpose: classical ML-based diagnostics. + +Potential capabilities: +- baseline models +- feature importance +- model diagnostics +- Isolation Forest +- LOF +- ECOD/COPOD/HBOS or equivalent anomaly methods +- optional gradient boosting diagnostics +- optional SHAP-style explanation layer + +This pack is diagnostic, not a general AutoML system. + +CLI/TUI label: **ML Diagnostics** + +--- + +### `deep` +Purpose: optional nonlinear/deep diagnostics where they add real value. + +Potential capabilities: +- MLP autoencoder anomaly detector +- DeepSVDD-style anomaly detector +- variational autoencoder experiment path +- small temporal CNN/TCN models +- optional tabular nonlinear baseline + +Framework policy: +- do not require a deep-learning framework in the core package +- choose one supported runtime initially rather than supporting everything +- CPU must remain supported for small models where practical +- GPU use should be opt-in/auto-detected and never required for ordinary FrameVitals analysis + +CLI/TUI label: **Deep Models** + +--- + +### `text` +Purpose: advanced text and semantic diagnostics. + +Potential capabilities: +- embedding-based text drift +- semantic similarity/grouping +- advanced PII helpers +- language detection +- text quality/duplication signals + +Large embedding weights should be managed by the model registry, not bundled directly into the wheel. + +CLI/TUI label: **Advanced Text / NLP** + +--- + +### `polars` +Purpose: Polars input/backend support. + +Potential capabilities: +- DataFrame/LazyFrame adapters +- projection/predicate-aware analysis paths where practical +- backend-native operations for large data + +CLI/TUI label: **Polars Backend** + +--- + +### `arrow` +Purpose: PyArrow and columnar I/O. + +Potential capabilities: +- Arrow Table/RecordBatch support +- Parquet scans +- batch/streaming analysis paths +- projection/filter support + +CLI/TUI label: **Arrow / Parquet Backend** + +--- + +### `sql` +Purpose: database/query-backed analysis. + +Potential capabilities: +- DuckDB adapter +- SQLAlchemy-compatible sources where useful +- pushdown of lightweight aggregates/projections + +The initial goal should be analysis of query results and large local files rather than becoming a database management tool. + +CLI/TUI label: **SQL Adapters** + +--- + +### `cloud` +Purpose: remote object/file storage integration. + +Potential capabilities: +- fsspec-based inputs +- S3-compatible paths +- selected cloud filesystem adapters + +Credentials must be supplied by the user's environment/provider tooling; FrameVitals should not invent its own secret store. + +CLI/TUI label: **Cloud Filesystems** + +--- + +### `tui` +Purpose: interactive terminal application if kept separate from core. + +Potential contents: +- terminal UI framework +- interactive tables/forms/progress views + +If the dependency footprint is small enough, this pack may eventually become part of the default install. Until then, the standard argparse CLI remains available without it. + +CLI/TUI label: **Interactive Terminal UI** + +--- + +### `ai` +Purpose: optional natural-language interpretation/agent features. + +Rules: +- deterministic analysis remains the source of factual metrics +- AI summarizes/interprets structured findings +- no AI dependency required for normal analysis +- provider-specific behavior stays isolated + +CLI/TUI label: **AI Interpretation** + +## Example extras layout + +Conceptually: + +```toml +[project.optional-dependencies] +viz = ["..."] +excel = ["..."] +ml = ["..."] +deep = ["..."] +text = ["..."] +polars = ["..."] +arrow = ["..."] +sql = ["..."] +cloud = ["..."] +tui = ["..."] +ai = ["..."] +``` + +The exact dependency choices should be benchmarked and reviewed before modifying `pyproject.toml`. + +## Capability registry + +FrameVitals should expose capability metadata independently from the Python packaging implementation. + +Example conceptual record: + +```python +Capability( + id="deep", + name="Deep Models", + installed=False, + enabled=False, + extra="deep", + provides=[ + "anomaly.autoencoder", + "anomaly.deep_svdd", + "timeseries.tcn", + ], + resource_class="high", +) +``` + +This registry powers: +- `framevitals addons list` +- TUI install/toggle screens +- planner applicability checks +- `framevitals doctor` +- error messages when an optional analysis was explicitly requested but is unavailable + +## Installed vs enabled + +These states must not be conflated. + +| Installed | Enabled | Meaning | +|---|---|---| +| No | No | capability unavailable | +| Yes | No | dependency exists but planner will not use it automatically | +| Yes | Yes | planner may use capability when applicable | +| No | Yes | invalid state; config should warn and treat as unavailable | + +Users should be able to keep a pack installed but disabled for speed/reproducibility. + +## Model registry + +Model files should have a separate lifecycle from Python packages. + +Suggested cache organization: + +```text +/ + models/ + semantic-types-small/ + 1.0.0/ + model... + manifest.json + temporal-anomaly-tiny/ + 1.0.0/ + model... + manifest.json +``` + +Each manifest should include: +- model ID/version +- FrameVitals compatibility +- task +- framework/runtime +- checksum +- source URL +- license +- file sizes +- expected input contract + +## Download policy + +Model download should occur only through explicit actions such as: + +```bash +framevitals models install semantic-types-small +``` + +or the equivalent TUI **Download** button. + +Ordinary analysis can suggest a model but should not automatically download it. + +Example: + +```text +Semantic type confidence is low for 3 columns. +Optional model 'semantic-types-small' could provide a second opinion. +[ Download & enable ] [ Ignore ] +``` + +Even there, user confirmation is required. + +## Reproducibility + +Every analysis result should record optional capability/model metadata when used: +- pack ID/version +- relevant dependency versions +- model ID/version/checksum +- backend +- random seed +- device (CPU/GPU) + +This makes deep/model-assisted diagnostics reproducible and explainable. + +## Dependency policy + +Before adding a package to any optional pack, evaluate: +1. user value +2. wheel size and install reliability +3. Python/platform compatibility +4. maintenance activity +5. license compatibility +6. startup/import cost +7. whether FrameVitals can implement the needed behavior with an existing dependency instead + +A large dependency should not enter the base install merely because one analysis can use it. diff --git a/docs/architecture-proposal/CLI_TUI_SPEC.md b/docs/architecture-proposal/CLI_TUI_SPEC.md new file mode 100644 index 0000000..5cde425 --- /dev/null +++ b/docs/architecture-proposal/CLI_TUI_SPEC.md @@ -0,0 +1,326 @@ +# FrameVitals CLI / TUI Specification + +This document defines the intended command-line and interactive terminal experience for future FrameVitals releases. + +## Goals + +The CLI should satisfy two very different users without forcing either into the other's workflow: + +1. **Interactive user** — wants to explore configuration, install optional capabilities, select analyses, and inspect reports through a terminal UI. +2. **Automation user** — wants stable commands, exit codes, JSON output, and no prompts for scripts/CI. + +The same underlying configuration and execution engine must power both experiences. + +## Entry behavior + +```bash +framevitals +``` + +If an interactive TTY is available and the TUI capability is installed, open the FrameVitals terminal application. + +If the TUI is unavailable, show normal CLI help and a short message explaining how to install the interactive capability. + +Explicit subcommands must never unexpectedly open the TUI: + +```bash +framevitals analyze data.csv +framevitals compare reference.csv current.csv +framevitals validate data.csv --contract contract.json +``` + +## Main navigation + +```text +FrameVitals + + Analyze Dataset + Compare Datasets + Validate Contract + Reports & Snapshots + Configuration + Add-ons + Models + Doctor + Help + Exit +``` + +## Analysis screen + +```text +Dataset: /Users/me/data/customers.csv +Target: churn + +Preset +( ) Quick (*) Standard ( ) Deep ( ) Exhaustive ( ) Custom + +Analysis categories +[x] Structural profile +[x] Data quality +[x] Statistics +[x] Relationships +[x] ML readiness +[x] Target intelligence +[x] Drift / comparison auto when reference exists +[x] Time-series analysis auto when applicable +[x] Text analysis auto when applicable +[x] Privacy / PII checks +[ ] Deep models optional pack +[ ] AI interpretation optional pack + +Resources +Backend Auto +Workers 4 +Memory limit 4 GB +Time limit 120 s +GPU Auto +Sampling Adaptive + +[ Show Plan ] [ Run Analysis ] [ Save Profile ] +``` + +## Plan screen + +Before expensive runs, users should be able to see what FrameVitals intends to execute. + +```text +Execution Plan + +RUN structural_profile essential ~low +RUN missingness_analysis essential ~low +RUN relationship_matrix high ~medium +RUN anomaly_ensemble high ~medium +SKIP tcn_temporal_anomaly not installed +SKIP text_embeddings no applicable text column +SKIP target_leakage no target selected + +Estimated class: medium +Expected peak memory: < 2 GB + +[ Run ] [ Configure ] [ Back ] +``` + +Every skipped item should have an explicit reason. + +## Add-on manager + +Optional functionality must be easy to discover and install from the TUI. + +```text +Add-ons + +Core +[x] Core analysis Built in +[x] Contracts Built in + +Optional +[x] Visual reports Installed Enabled +[ ] Excel support Not installed [ Install ] +[x] ML diagnostics Installed Enabled +[ ] Deep models Not installed [ Install ] +[ ] Advanced text/NLP Not installed [ Install ] +[ ] Polars backend Not installed [ Install ] +[ ] Arrow backend Not installed [ Install ] +[ ] SQL adapters Not installed [ Install ] +[ ] Cloud filesystems Not installed [ Install ] + +Space Toggle enabled state +Enter Install / Open details +R Remove +``` + +### Installation flow + +Selecting `Install` should show exactly what will happen: + +```text +Install: Deep Models + +Provides +- Autoencoder anomaly detector +- DeepSVDD anomaly detector +- Optional temporal CNN/TCN diagnostics + +Python packages to install + framevitals[deep] + +Estimated package download: 220 MB +Model downloads: none until a model is explicitly selected + +Command + /path/to/python -m pip install "framevitals[deep]" + +[ Install ] [ Cancel ] +``` + +Rules: +- Never install implicitly during `import framevitals`. +- Never install from a normal non-interactive analysis command. +- Interactive install requires confirmation. +- Show which Python environment will be modified. +- Display installation errors instead of hiding them. +- Allow user to copy the command and run it manually. + +## Model manager + +Python add-ons and model weights are different resources and should be managed separately. + +```text +Models + +Semantic Type Classifier +[ ] semantic-types-small 24 MB [ Download ] + +Text +[ ] text-embeddings-small 90 MB [ Download ] + +Time Series +[ ] temporal-anomaly-tiny 18 MB [ Download ] + +Anomaly +[ ] tabular-ae-reference 12 MB [ Download ] +``` + +Model registry metadata should include: +- model ID +- model version +- compatible FrameVitals versions +- task +- file size +- checksum +- license +- source +- framework/runtime requirement +- expected hardware + +Downloaded weights should be cached in an OS-appropriate FrameVitals data directory, not in the working project directory. + +## Configuration screen + +Configuration should be editable interactively but stored in a human-readable file. + +Suggested conceptual config: + +```toml +[analysis] +preset = "standard" +statistics = true +relationships = true +ml_readiness = true +deep_models = false +ai_interpretation = false + +[resources] +workers = 4 +max_memory = "4GB" +max_time_seconds = 120 +gpu = "auto" +sampling = "adaptive" + +[reporting] +terminal = true +html = false +json = false + +[privacy] +pii_detection = true +show_raw_sensitive_examples = false +``` + +## Configuration precedence + +Highest priority wins: + +1. Explicit Python arguments / command flags +2. Project configuration (`framevitals.toml`) +3. User configuration +4. Environment variables +5. Selected preset +6. Built-in defaults + +`framevitals config explain` should show where each effective value came from. + +## Scriptable command design + +Interactive actions should have deterministic command equivalents. + +```bash +# analysis +framevitals analyze data.csv --preset deep +framevitals analyze data.csv --target churn --html report.html --json report.json +framevitals plan data.csv --preset exhaustive --explain + +# configuration +framevitals config show +framevitals config set analysis.deep_models true +framevitals config profile use laptop +framevitals config explain + +# add-ons +framevitals addons list +framevitals addons install deep +framevitals addons enable deep +framevitals addons disable deep +framevitals addons remove deep + +# models +framevitals models list +framevitals models install semantic-types-small +framevitals models remove semantic-types-small + +# environment diagnostics +framevitals doctor +framevitals doctor --json +``` + +## Exit codes + +Suggested stable policy: +- `0` — command completed / validation passed +- `1` — validation or configured quality gate failed +- `2` — invalid arguments/configuration +- `3` — input/data loading failure +- `4` — optional capability unavailable +- `5` — internal execution failure + +Exact values can change before stabilization, but the policy should become documented and tested before 1.0. + +## Non-interactive safety + +When stdin/stdout is not interactive: +- never prompt +- never install packages +- never download models unless an explicit install command was invoked +- emit deterministic output +- respect `--json`/`--quiet` +- use stable exit codes + +## Doctor screen + +`framevitals doctor` should answer common environment questions: + +```text +FrameVitals 0.x +Python 3.13.7 +Platform macOS arm64 +Core OK +ML pack Installed +Deep pack Missing +Polars Missing +PyArrow Installed +GPU runtime Not detected +Cache /Users/me/Library/Caches/framevitals +Config ~/.config/framevitals/config.toml + +No blocking problems detected. +``` + +## UX principle + +The TUI should hide accidental complexity, not analytical detail. Users should be able to make common decisions with a toggle or button while still being able to inspect: +- exactly what analysis is being run +- why it was selected +- what it costs +- which optional dependency/model it uses +- what command/change an install button performs diff --git a/docs/architecture-proposal/DECISIONS.md b/docs/architecture-proposal/DECISIONS.md new file mode 100644 index 0000000..396b0a3 --- /dev/null +++ b/docs/architecture-proposal/DECISIONS.md @@ -0,0 +1,136 @@ +# FrameVitals Architecture Decisions + +This file records decisions that were explicitly accepted during architecture planning so future implementation does not repeatedly reopen the same product questions without new evidence. + +These are direction-setting principles, not a frozen public API contract. Current package behavior and stability guarantees are defined by the maintained code, tests, and release documentation. + +## D-001 — Exhaustive capability, selective execution + +**Status:** Accepted + +FrameVitals may aim for a very broad and eventually exhaustive catalog of tabular-data diagnostics. A normal analysis must not run everything blindly. + +Default execution is selected by: +- data applicability +- preset/configuration +- installed and enabled capabilities +- resource budget +- analysis dependencies + +`exhaustive` means every applicable installed analysis permitted by the active resource policy. + +--- + +## D-002 — Interactive terminal UI plus scriptable CLI + +**Status:** Accepted direction + +Running `framevitals` with no explicit subcommand may eventually open an interactive terminal application when that capability is installed. + +Explicit commands such as `framevitals analyze`, `compare`, and `validate` remain deterministic and suitable for scripts/CI. + +The TUI and CLI should share the same configuration and execution engine. + +--- + +## D-003 — Optional features use install actions/toggles + +**Status:** Accepted direction + +Optional capabilities should be discoverable rather than forcing users to research dependency groups manually. + +Any interactive installation action must still show the exact command/environment being modified and request confirmation. + +Installed and enabled are separate states. + +--- + +## D-004 — No implicit package/model downloads + +**Status:** Accepted + +FrameVitals never installs Python packages or downloads model weights during normal import or ordinary analysis. + +Downloads occur only through explicit add-on/model installation actions or equivalent dedicated commands. + +--- + +## D-005 — Deep learning is optional and task-specific + +**Status:** Accepted + +ML/DL is used only where it adds analytical value, for example: +- nonlinear multivariate anomaly detection +- temporal CNN/TCN diagnostics for ordered time-series windows +- optional semantic-type second opinion +- embedding-based text drift +- nonlinear learnability diagnostics + +CNNs are not applied to generic unordered tabular rows merely because they are available. + +Deterministic/statistical fallbacks remain first-class. + +--- + +## D-006 — FrameVitals is not an AutoML platform + +**Status:** Accepted + +Baseline models and nonlinear diagnostic models can measure learnability, leakage, feature behavior, or explainability. The product does not optimize around training/deploying a production model fleet. + +The central question remains: **what is true about this dataset, what is wrong, what changed, and what should the user investigate next?** + +--- + +## D-007 — Result/report UX precedes algorithm count + +**Status:** Accepted + +Stable result objects/schema, normalized findings, full JSON output, good terminal rendering, and useful reports are higher priority than continuously adding isolated algorithms. + +New analysis is only valuable when users can consume and trust the result. + +--- + +## D-008 — One reusable analysis context / exact-once facts + +**Status:** Accepted direction + +Modules should consume reusable facts from the execution context rather than independently rescanning the same dataset for missingness, dtypes, semantic roles, samples, moments, correlations, and related facts. + +When the source pass has already established an exact sufficient statistic, downstream diagnostics should reuse it rather than replace it with a bounded-sample estimate. + +--- + +## D-009 — Cleaning is plan-first + +**Status:** Accepted direction + +FrameVitals may recommend/simulate cleaning operations. It should not silently mutate user data. + +Preferred workflow: +1. detect issue +2. propose cleaning action +3. estimate/simulate impact when possible +4. user explicitly applies approved transformations +5. record an audit trail + +--- + +## D-010 — Public project website is separate from analysis runtime + +**Status:** Accepted + +The public website is primarily for product explanation, demos, installation, documentation, and benchmark/transparency information. + +It does not need to become a dataset-upload service. The website and generated reports can share a coherent visual language while remaining separate products. + +--- + +## D-011 — Planning material is preserved but not release contract + +**Status:** Accepted + +Architecture proposal material is preserved under `docs/architecture-proposal/` in the integration history. It should not be treated as the current release contract. + +Implementation changes still land through normal development/review and must earn their place through correctness, resource, compatibility, and benchmark evidence. diff --git a/docs/architecture-proposal/IMPLEMENTATION_ROADMAP.md b/docs/architecture-proposal/IMPLEMENTATION_ROADMAP.md new file mode 100644 index 0000000..2e75725 --- /dev/null +++ b/docs/architecture-proposal/IMPLEMENTATION_ROADMAP.md @@ -0,0 +1,274 @@ +# FrameVitals Implementation Roadmap + +This document turns the future architecture proposal into a practical sequence of pull requests. It is a planning reference, not a promise that every item ships in the named release. + +## Operating rule + +FrameVitals should become **exhaustive in capability and selective in execution**. New capabilities should be added behind stable interfaces, configuration, applicability checks, and resource budgets rather than wired directly into one ever-growing pipeline. + +## Build order + +### PR 1 — Result model foundation +**Priority:** P0 + +Create a stable internal/public result layer without breaking the current dictionary-returning API. + +Proposed modules: +- `src/framevitals/result.py` +- `src/framevitals/findings.py` +- `src/framevitals/metadata.py` + +Deliverables: +- `AnalysisResult` or an internal result object with mapping compatibility. +- Normalized `Finding` structure: code, title, severity, confidence, evidence, affected columns, recommendation, method. +- `to_dict()` and JSON-safe serialization. +- Existing `fv.analyze()` output remains backward compatible during migration. + +Acceptance criteria: +- Existing tests still pass. +- Old dictionary access remains available. +- Result schema receives an explicit version. + +--- + +### PR 2 — Full report export and terminal renderer +**Priority:** P0 + +Fix the current gap between the rich Python result and the summary-only CLI output. + +Proposed modules: +- `src/framevitals/reporting/terminal.py` +- `src/framevitals/reporting/json.py` + +Deliverables: +- Human-friendly terminal summary. +- Full JSON export from CLI. +- Clear distinction between terminal summary and machine-readable full result. +- Stable exit behavior. + +Acceptance criteria: +- `framevitals analyze data.csv --output report.json` can intentionally write the full report. +- Terminal output is concise and does not dump enormous nested JSON by default. + +--- + +### PR 3 — Standalone HTML report +**Priority:** P0 + +Proposed modules: +- `src/framevitals/reporting/html.py` +- `src/framevitals/reporting/assets.py` + +Deliverables: +- Self-contained HTML report requiring no server. +- Overview, health, findings, missingness, column profiles, relationships, anomalies, target intelligence, drift when available, and recommendations. +- Deterministic report generation from an existing result object. + +Acceptance criteria: +- Report works offline in a browser. +- No user dataset is uploaded anywhere. +- HTML generation does not re-run analysis. + +--- + +### PR 4 — Configuration and presets +**Priority:** P0 + +Proposed modules: +- `src/framevitals/config.py` +- `src/framevitals/config_presets.py` +- `src/framevitals/config_profiles.py` + +Deliverables: +- Typed configuration object. +- Presets: `quick`, `standard`, `deep`, `exhaustive`, `custom`. +- Resource policy: workers, memory budget, time budget, sampling policy, GPU preference. +- Analysis-category toggles. +- Config precedence rules. + +Suggested precedence: +1. explicit Python/CLI arguments +2. project config +3. user config +4. environment variables +5. preset defaults + +Acceptance criteria: +- Same config can be used from Python and CLI. +- Config validation produces useful errors. + +--- + +### PR 5 — AnalysisContext and reusable cache +**Priority:** P0 + +Proposed modules: +- `src/framevitals/execution/context.py` +- `src/framevitals/execution/cache.py` +- `src/framevitals/execution/sampling.py` + +Deliverables: +- One context per run containing loaded data/backend, shape, profile, roles, fingerprints, reusable samples, configuration, seed, installed capabilities, and cached intermediates. +- Modules request reusable facts from the context rather than rescanning the dataset. + +Acceptance criteria: +- Current outputs remain equivalent for deterministic analyses. +- Benchmarks demonstrate fewer full-data scans on representative datasets. + +--- + +### PR 6 — Planner-controlled execution +**Priority:** P0 + +Evolve `analysis_inventory.py` + `analysis_selector.py` into a real execution planner. + +Proposed modules: +- `src/framevitals/planner.py` +- `src/framevitals/execution/budget.py` + +Deliverables: +- Registry entries declare inputs, applicability, cost, dependencies, outputs, optional pack, and resource class. +- Planner produces an explainable execution plan. +- `framevitals plan data.csv --explain` shows what will run and why. +- Exhaustive mode means every applicable installed analysis that fits the configured policy. + +Acceptance criteria: +- The planner actually controls execution rather than only describing it. +- Skipped analyses record a reason. + +--- + +### PR 7 — Interactive CLI/TUI foundation +**Priority:** P1 + +Proposed modules: +- `src/framevitals/cli_tui.py` +- `src/framevitals/doctor.py` + +Deliverables: +- Running `framevitals` without a subcommand opens the interactive interface when the TUI is available. +- Analyze, Compare, Validate, Reports, Configuration, Add-ons, Models, Doctor. +- Traditional subcommands remain fully usable for scripts and CI. + +Acceptance criteria: +- No TUI dependency is imported during a normal library import. +- Non-interactive behavior remains stable. + +--- + +### PR 8 — Add-on manager and capability registry +**Priority:** P1 + +Proposed modules: +- `src/framevitals/addons.py` +- `src/framevitals/capabilities.py` +- `src/framevitals/models/registry.py` + +Deliverables: +- Installed/enabled are separate states. +- Discover whether optional dependencies are installed. +- TUI install/remove/toggle actions. +- Scriptable equivalents: `addons list/install/remove/enable/disable`. +- Exact command displayed before modifying the active environment. + +Acceptance criteria: +- Importing FrameVitals never installs anything. +- Installation requires explicit confirmation in interactive mode. + +--- + +### PR 9 — Semantic types and deeper profiling +**Priority:** P1 + +Proposed modules: +- `src/framevitals/semantic_types.py` +- `src/framevitals/quality/keys.py` +- `src/framevitals/quality/missingness.py` + +Deliverables: +- Stronger semantic types: ID, UUID, email, URL, IP, phone-like, currency, percentage, geospatial-like, date/time, free text, code/category. +- Candidate primary keys and composite-key hints. +- Missingness patterns and co-occurrence. +- Categorical normalization suggestions. +- Optional small character model can later act as a second opinion when rule confidence is low. + +--- + +### PR 10 — Target intelligence and relationship engine +**Priority:** P1 + +Proposed modules: +- `src/framevitals/target_intelligence.py` +- `src/framevitals/relationships.py` +- `src/framevitals/ml/label_quality.py` + +Deliverables: +- Target/task inference. +- Class imbalance and target quality. +- Proxy, temporal, and split leakage. +- Numeric↔numeric, categorical↔categorical, numeric↔categorical relationships. +- Feature stability/redundancy signals. + +Optional model support: +- nonlinear baseline as a diagnostic, not AutoML. + +--- + +### PR 11 — Drift, snapshots, and monitoring foundation +**Priority:** P1 + +Proposed modules: +- `src/framevitals/drift/` +- `src/framevitals/monitoring/snapshot.py` +- `src/framevitals/monitoring/store.py` + +Deliverables: +- Pluggable drift methods and automatic method selection. +- Drift severity and confidence/evidence. +- Schema diff. +- Result snapshots and local history. +- Compare current result against a baseline snapshot. + +--- + +### PR 12 — Scale/backends and first optional model pack +**Priority:** P2 + +Proposed modules: +- `src/framevitals/backends/` +- `src/framevitals/io/` +- `src/framevitals/models/deep_anomaly.py` +- `src/framevitals/models/runtime.py` + +Deliverables: +- Polars/PyArrow/Narwhals strategy. +- Parquet and chunked/streaming paths. +- Large-data sampling/budget behavior. +- Optional anomaly pack: autoencoder and/or DeepSVDD alongside classical detectors. +- Optional GPU selection only when useful and installed. + +Acceptance criteria: +- Core install stays usable without deep-learning frameworks. +- Model pack failure never removes deterministic fallback analysis. + +## Parallel workstreams + +The public project website can be developed in parallel because it should initially use curated/precomputed demo data rather than depend on the full backend roadmap. + +Recommended parallel streams: +- **Core:** PR 1–6 +- **Experience:** PR 2–3, 7–8 +- **Analysis depth:** PR 9–11 +- **Scale/models:** PR 12+ +- **Website:** independent visual/demo frontend with shared report design language + +## Release philosophy + +Do not ship a release because the project gained a certain number of algorithms. Ship when a coherent user workflow becomes materially better. + +Examples: +- result/report release +- configuration/planner release +- interactive/add-on release +- monitoring release +- scale/backend release diff --git a/docs/architecture-proposal/PROPOSED_MODULE_TREE.md b/docs/architecture-proposal/PROPOSED_MODULE_TREE.md new file mode 100644 index 0000000..0559b74 --- /dev/null +++ b/docs/architecture-proposal/PROPOSED_MODULE_TREE.md @@ -0,0 +1,256 @@ +# Proposed Future Module Tree + +This is a destination architecture, **not** a request to rename/move every current module immediately. Existing public imports should remain compatible while internal organization evolves gradually. + +```text +src/framevitals/ +├── __init__.py +├── api.py +├── result.py +├── findings.py +├── metadata.py +├── errors.py +├── config.py +├── config_presets.py +├── config_profiles.py +├── capabilities.py +├── addons.py +├── doctor.py +│ +├── execution/ +│ ├── __init__.py +│ ├── context.py +│ ├── planner.py +│ ├── inventory.py +│ ├── budget.py +│ ├── cache.py +│ ├── sampling.py +│ └── scheduler.py +│ +├── io/ +│ ├── __init__.py +│ ├── loader.py +│ ├── csv.py +│ ├── excel.py +│ ├── json.py +│ ├── parquet.py +│ ├── sql.py +│ └── cloud.py +│ +├── backends/ +│ ├── __init__.py +│ ├── base.py +│ ├── pandas.py +│ ├── polars.py +│ ├── arrow.py +│ └── narwhals_adapter.py +│ +├── profiling/ +│ ├── __init__.py +│ ├── structural.py +│ ├── semantic_types.py +│ ├── column_roles.py +│ ├── dataset_signals.py +│ ├── keys.py +│ ├── missingness.py +│ └── freshness.py +│ +├── quality/ +│ ├── __init__.py +│ ├── health.py +│ ├── duplicates.py +│ ├── consistency.py +│ ├── ranges.py +│ ├── categories.py +│ └── pii.py +│ +├── contracts/ +│ ├── __init__.py +│ ├── infer.py +│ ├── validate.py +│ ├── rules.py +│ ├── policy.py +│ ├── diff.py +│ └── schema.py +│ +├── statistics/ +│ ├── __init__.py +│ ├── descriptive.py +│ ├── distributions.py +│ ├── robust.py +│ ├── testing.py +│ ├── relationships.py +│ ├── multicollinearity.py +│ └── segments.py +│ +├── anomaly/ +│ ├── __init__.py +│ ├── ensemble.py +│ ├── univariate.py +│ ├── classical.py +│ └── scoring.py +│ +├── drift/ +│ ├── __init__.py +│ ├── compare.py +│ ├── numeric.py +│ ├── categorical.py +│ ├── text.py +│ ├── schema.py +│ ├── methods.py +│ └── selection.py +│ +├── target/ +│ ├── __init__.py +│ ├── intelligence.py +│ ├── task.py +│ ├── leakage.py +│ ├── imbalance.py +│ ├── label_quality.py +│ └── candidates.py +│ +├── ml/ +│ ├── __init__.py +│ ├── readiness.py +│ ├── preprocessing.py +│ ├── baseline.py +│ ├── leaderboard.py +│ ├── diagnostics.py +│ ├── feature_importance.py +│ └── explainability.py +│ +├── time_series/ +│ ├── __init__.py +│ ├── detection.py +│ ├── quality.py +│ ├── diagnostics.py +│ ├── seasonality.py +│ ├── change_points.py +│ └── anomaly.py +│ +├── text/ +│ ├── __init__.py +│ ├── profile.py +│ ├── quality.py +│ ├── duplication.py +│ ├── pii.py +│ └── drift.py +│ +├── cleaning/ +│ ├── __init__.py +│ ├── plan.py +│ ├── suggestions.py +│ ├── simulate.py +│ ├── apply.py +│ └── audit.py +│ +├── monitoring/ +│ ├── __init__.py +│ ├── snapshot.py +│ ├── store.py +│ ├── history.py +│ └── policy.py +│ +├── models/ +│ ├── __init__.py +│ ├── registry.py +│ ├── manifest.py +│ ├── runtime.py +│ ├── semantic_types.py +│ ├── deep_anomaly.py +│ ├── temporal.py +│ └── embeddings.py +│ +├── reporting/ +│ ├── __init__.py +│ ├── terminal.py +│ ├── json.py +│ ├── html.py +│ ├── pdf.py +│ ├── notebook.py +│ ├── charts.py +│ ├── compare_html.py +│ └── privacy.py +│ +├── plugins/ +│ ├── __init__.py +│ ├── registry.py +│ ├── checks.py +│ ├── analyzers.py +│ └── hooks.py +│ +├── ai/ +│ ├── __init__.py +│ ├── insights.py +│ ├── agent.py +│ ├── tools.py +│ └── brief.py +│ +├── cli.py +└── cli_tui.py +``` + +## Migration philosophy + +Do **not** perform a giant move-only refactor immediately. Prefer: + +1. introduce a new namespace when implementing a new coherent capability +2. move an old module only when it is actively being changed or causes architectural friction +3. keep compatibility shims for public imports where necessary +4. delete shims only through documented deprecation cycles + +## Mapping from important current modules + +| Current module | Future home/direction | +|---|---| +| `pipeline.py` | gradually replaced/orchestrated by `execution/context.py` + `execution/planner.py` | +| `analysis_inventory.py` | `execution/inventory.py` | +| `analysis_selector.py` | `execution/planner.py` | +| `profiler.py` | `profiling/structural.py` | +| `column_roles.py` | `profiling/column_roles.py` + semantic typing layer | +| `dataset_signals.py` | `profiling/dataset_signals.py` | +| `health_score.py` | `quality/health.py` | +| `contracts.py` | split gradually into `contracts/` package | +| `drift_analysis.py` | split gradually into `drift/` package | +| `anomaly_ensemble.py` | `anomaly/ensemble.py` | +| `ml_readiness.py` | `ml/readiness.py` | +| `target_analyzer.py` | `target/intelligence.py` | +| `target_leakage.py` | `target/leakage.py` | +| `time_series.py` | split into `time_series/` package | +| `text_profile.py` | `text/profile.py` | +| `cleaner.py` | replaced by plan/simulate/apply/audit workflow in `cleaning/` | +| `visualizer.py` | `reporting/charts.py` | +| PDF/report modules | `reporting/` | +| `ai_agent.py`, `ai_insights.py`, `agent_tools.py`, `agent_brief.py` | isolated under optional `ai/` namespace | + +## Dependency direction + +A useful dependency rule is: + +```text +api/result/reporting + ↓ +execution planner/context + ↓ +analysis domains + ↓ +profiling/backends/config +``` + +Analysis-domain modules should **not** import the CLI/TUI or website code. Reporting should consume result structures rather than cause analyses to rerun. Optional AI should consume structured findings/results rather than become a required dependency of deterministic modules. + +## Plugin boundary + +Long term, third-party analysis plugins should register through a stable plugin API instead of monkey-patching the pipeline. + +A plugin should declare: +- ID/name/version +- supported FrameVitals plugin API version +- applicability predicate +- required inputs/context facts +- resource class +- optional dependency requirements +- output/finding schema +- whether failure is fatal or optional + +This allows FrameVitals to be exhaustive without requiring every niche analysis to live in the core repository forever. diff --git a/docs/architecture-proposal/README.md b/docs/architecture-proposal/README.md new file mode 100644 index 0000000..89fb461 --- /dev/null +++ b/docs/architecture-proposal/README.md @@ -0,0 +1,24 @@ +# FrameVitals Architecture Proposal Archive + +These documents were transferred from the former `docs/architecture-proposal` working branch into `develop/august` so the useful planning work can be preserved before branch cleanup. + +They are **planning references, not the current public API or release contract**. Several originally proposed items have already evolved into implemented FrameVitals features, including result objects, source-aware execution, execution budgets, streaming/Arrow support, native Rust kernels, planning APIs, monitoring snapshots, quality gates, and capability-oriented optional dependencies. + +For current behavior, prefer the maintained documentation in the parent `docs/` directory and the package code/tests. + +## Preserved planning documents + +- `IMPLEMENTATION_ROADMAP.md` — original staged implementation sequence and product philosophy. +- `BENCHMARK_ACCEPTANCE_PLAN.md` — correctness/performance/memory/backend acceptance ideas. +- `CAPABILITY_PACKS.md` — optional capability-pack and dependency-management design. +- `CLI_TUI_SPEC.md` — future interactive terminal design ideas. +- `DECISIONS.md` — accepted long-term product/architecture principles. +- `PROPOSED_MODULE_TREE.md` — proposed modular organization for future growth. + +Generated PDF/LaTeX artifacts and the proposal-only build workflow were intentionally **not** transferred. Source planning content belongs in Git; generated planning artifacts do not need to remain on an active development branch. + +## Core rule retained from the proposal + +> FrameVitals should be exhaustive in capability, not exhaustive in what it runs by default. + +That principle remains compatible with the current direction: the execution engine should choose the cheapest statistically defensible work based on applicability, scale, installed capabilities, backend availability, and requested analysis depth. diff --git a/docs/execution-provenance.md b/docs/execution-provenance.md new file mode 100644 index 0000000..609318e --- /dev/null +++ b/docs/execution-provenance.md @@ -0,0 +1,119 @@ +# Execution provenance + +FrameVitals results can mix exact metadata, full-stream calculations, bounded row +samples, estimates, and operations that intentionally materialize a complete pandas +DataFrame. The `execution` block makes those decisions machine-readable. + +## Schema version + +The shared execution contract currently uses: + +```json +{ + "execution_schema_version": "1" +} +``` + +During the `0.x` series, older operation-specific keys remain available while public +results converge on this common vocabulary. + +## Common fields + +| Field | Meaning | +| --- | --- | +| `execution_schema_version` | Version of the shared execution metadata contract | +| `method` | Stable description of the execution strategy | +| `full_materialization` | Whether FrameVitals created a complete pandas representation of a non-pandas source | +| `source` | Source metadata/capabilities when available | +| `sampled` | Whether row-level work used fewer than all source rows | +| `source_rows` | Exact source row count when known | +| `source_columns` | Exact source column count when known | +| `sample_rows` | Number of rows used for bounded row-level work | +| `strategy` | Sampling or source-consumption strategy | +| `components` | Per-component exactness/approximation information | +| `reason` | Human-readable explanation for the execution choice | +| `scope` | Legacy/operation-specific scope retained during the `0.x` migration | + +Not every field appears in every result. Missing information is omitted rather than +represented as ambiguous placeholder values. + +## Full materialization semantics + +`full_materialization` describes **what the operation did**, not merely where the +source came from. + +A pandas `DataFrame` is already materialized when supplied by the caller, so running +an exact operation on it does not count as FrameVitals materializing a new source. +By contrast, converting any of these into a complete pandas DataFrame does count: + +- a file-backed source; +- a PyArrow table; +- a DuckDB relation; +- a remote/custom source. + +This distinction matters for memory planning and for pipelines that deliberately stay +on Arrow/relation-backed execution paths. + +## Examples + +### Bounded statistics + +```json +{ + "execution_schema_version": "1", + "method": "bounded_deep_statistics", + "full_materialization": false, + "sampled": true, + "source_rows": 120000, + "source_columns": 18, + "sample_rows": 1000, + "strategy": "streaming_evenly_spaced_global_rows" +} +``` + +### Exact contract validation + +```json +{ + "execution_schema_version": "1", + "method": "exact_contract_validation", + "full_materialization": true, + "sampled": false, + "source": { + "kind": "relation", + "format": "duckdb" + } +} +``` + +### Quality gate + +A gate aggregates the execution blocks of the selected check families: + +```python +result = fv.gate( + current, + reference=reference, + contract=contract, + custom_checks=[positive_revenue], +) + +print(result["execution"]) +``` + +The top-level gate reports `full_materialization=True` when **any** selected check +family required it, while nested `validation`, `drift`, and `custom` entries preserve +family-level provenance. + +## Exactness is diagnostic-specific + +A streaming-capable source does not make every metric approximate. For example: + +- source shape and schema can be exact; +- missingness can be computed over the full stream; +- duplicate rate may become a bounded estimate on a large source; +- deep statistics may use a deterministic bounded sample; +- contract uniqueness checks stay exact and therefore may materialize. + +Consumers should prefer the `execution`/`components` fields over assumptions based on +file format or dataset size. diff --git a/docs/extending-framevitals.md b/docs/extending-framevitals.md new file mode 100644 index 0000000..6675782 --- /dev/null +++ b/docs/extending-framevitals.md @@ -0,0 +1,163 @@ +# Extending FrameVitals + +FrameVitals is designed to be extended at the edges rather than forked in the core. +There are two intentionally small extension boundaries: + +1. **custom data checks** for domain invariants; +2. **`DatasetSource` implementations** for new storage/query systems. + +Third-party check packages can additionally register checks through standard Python +entry points. Source adapters are normal Python objects implementing the source +protocol and can be shipped by any package without FrameVitals importing that package. + +## Custom checks + +Use `@framevitals.check` for application-owned invariants: + +```python +import framevitals as fv + +@fv.check( + "positive revenue", + severity="error", + description="Revenue cannot be negative.", +) +def positive_revenue(df): + minimum = float(df["revenue"].min()) + return { + "passed": minimum >= 0, + "message": "Negative revenue found." if minimum < 0 else "Revenue is valid.", + "details": {"minimum": minimum}, + } +``` + +A check may return either a boolean-like value or a mapping containing `passed`. +Mappings can also include `message` and `details`. + +Custom checks intentionally run against the complete DataFrame. FrameVitals cannot +safely infer whether an arbitrary Python invariant is sampleable, so it does not +silently weaken user-defined rules to bounded samples. + +## Publishing a check plugin + +A third-party package can expose checks through the `framevitals.checks` entry-point +group: + +```toml +[project.entry-points."framevitals.checks"] +positive_revenue = "acme_framevitals_checks:positive_revenue" +``` + +The exported object can be a `DataCheck` or a compatible DataFrame callable. +Applications opt in explicitly: + +```python +checks = fv.discover_checks() +result = fv.gate(current, custom_checks=checks) +``` + +FrameVitals never auto-loads installed check plugins. Loading an entry point executes +provider code, so discovery is an explicit trust decision. + +### Plugin guidance + +A good provider package should: + +- keep public check names stable; +- avoid network or filesystem side effects during module import; +- return JSON-friendly `details` values; +- document whether a rule is a warning or a hard error; +- test against the supported FrameVitals version range; +- avoid mutating the DataFrame passed into a check. + +FrameVitals gives every check an isolated DataFrame copy, but provider code should +still behave as a pure predicate where practical. + +## Custom DatasetSource implementations + +A storage/query integration does not need to be added to FrameVitals core. The minimal +protocol is deliberately small: + +```python +from framevitals.sources import DatasetMetadata + +class MySource: + def inspect(self): + return DatasetMetadata( + name="warehouse.orders", + kind="remote", + format="my-engine", + rows=1_000_000, + columns=12, + size_bytes=None, + materialized=False, + supports_projection=False, + supports_streaming=False, + ) + + def load(self): + # Return the complete dataset as pandas only when an operation requires it. + return fetch_as_pandas() +``` + +Then pass the source directly: + +```python +source = MySource() +print(fv.inspect_source(source)) +report = fv.analyze(source, mode="quick") +``` + +Because the protocol is runtime-checkable, FrameVitals can consume compatible custom +objects without maintaining a registry of every provider package. + +## Streaming source protocol + +To opt into source-aware batch execution, additionally implement `iter_batches`: + +```python +class MyStreamingSource(MySource): + def iter_batches(self, *, batch_size=65_536, columns=None): + ... +``` + +A streaming adapter should preserve these invariants: + +1. `inspect()` reports the **true source shape**, not the retained sample shape. +2. `iter_batches()` respects `batch_size` and requested projection where advertised. +3. yielded batches are compatible with the Arrow-oriented streaming profiler. +4. `load()` remains available for exact/full-row APIs. +5. optional provider dependencies are imported lazily. +6. the adapter does not pretend an exact operation is streaming merely because the + source supports batches. + +## Prefer standard interoperability over one-off adapters + +If a data library exposes the Arrow C Stream / PyCapsule interface, FrameVitals can +usually consume it through the generic Arrow boundary instead of carrying a dedicated +library dependency. + +That is how FrameVitals can interoperate with modern dataframe producers such as +Polars while keeping Polars out of the runtime dependency graph. + +Use a dedicated adapter when the storage/query system provides capabilities that the +generic Arrow boundary would lose. The DuckDB adapter is an example: it preserves a +lazy relation, pushes column projection into DuckDB, obtains exact relation metadata, +and streams Arrow batches without first constructing a complete in-memory table. + +## Execution provenance for extensions + +Extension authors should not invent incompatible exactness terminology. FrameVitals' +shared execution schema uses fields such as: + +- `method` +- `full_materialization` +- `sampled` +- `source_rows` +- `source_columns` +- `sample_rows` +- `strategy` +- `components` +- `reason` + +See [Execution provenance](execution-provenance.md) for the current schema. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..babd9fa --- /dev/null +++ b/docs/index.md @@ -0,0 +1,99 @@ +# FrameVitals + +**FrameVitals is a source-aware data-health and quality-gate engine for tabular pipelines.** + +It provides one workflow for inspecting data health, comparing production batches, +validating contracts, enforcing domain-specific invariants, and retaining compact +monitoring history. + +```python +import framevitals as fv + +source = fv.inspect_source("production.parquet") +report = fv.analyze("production.parquet", mode="quick") +drift = fv.compare("training.parquet", "production.parquet") +validation = fv.validate("production.parquet", contract) +gate = fv.gate( + "production.parquet", + reference="training.parquet", + contract=contract, +) +``` + +## Core design + +FrameVitals is built around four constraints: + +1. **Source-aware execution** — supported sources expose metadata, projection, and + batches before FrameVitals decides whether pandas materialization is necessary. +2. **Bounded expensive work** — statistics, anomaly detection, relationship discovery, + and drift use explicit execution budgets when full-row computation is not required. +3. **Exactness where correctness requires it** — contracts and arbitrary custom Python + invariants are not silently weakened to sampled checks. +4. **Execution transparency** — public results disclose whether work was exact, + sampled, estimated, streamed, or fully materialized. + +## Main workflow + +```text +source + │ + ├─► inspect_source + │ + ├─► analyze ─► snapshot ─► SnapshotHistory + │ + ├─► compare(reference, current) + │ + ├─► validate(current, contract) + │ + └─► gate + ├─ contract validation + ├─ drift + └─ custom checks / plugins +``` + +## Supported source families + +FrameVitals currently recognizes: + +- pandas `DataFrame` inputs; +- CSV and TSV files, with optional Arrow streaming; +- Parquet through the Arrow capability; +- PyArrow `Table` and `RecordBatch` inputs; +- table producers supporting the Arrow C Stream / PyCapsule interface; +- lazy DuckDB relations through the optional DuckDB adapter; +- custom objects implementing the FrameVitals `DatasetSource` protocol. + +Use `fv.inspect_source(data)` before analysis when you want to see the source's +shape metadata and streaming/projection capabilities. + +## Installation + +```bash +pip install framevitals +``` + +Optional capabilities are deliberately separated: + +```bash +pip install "framevitals[arrow]" +pip install "framevitals[duckdb]" +pip install "framevitals[excel]" +pip install "framevitals[plot]" +pip install "framevitals[ml]" +pip install "framevitals[ai]" +pip install "framevitals[web]" +``` + +For documentation development: + +```bash +pip install -e ".[docs]" +mkdocs serve +``` + +## Documentation map + +- [Source-aware execution](source-execution.md) +- [Execution provenance](execution-provenance.md) +- [Quality gates and custom checks](quality-gates.md) diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..f9f45a7 --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,120 @@ +# Monitoring with snapshots + +FrameVitals snapshots turn a full analysis result into compact, versioned monitoring +state. They are designed for workflows where you want to detect meaningful changes +without storing every raw production batch. + +## Create a snapshot in Python + +```python +import framevitals as fv + +report = fv.analyze("production.parquet", mode="quick") +snapshot = report.snapshot("production.snapshot.json") +``` + +A snapshot retains compact state such as: + +- source identity; +- schema and dtypes; +- missingness; +- duplicate rate; +- health score; +- ML-readiness score; +- finding codes; +- analysis configuration; +- a deterministic state fingerprint. + +It does **not** embed the full raw dataset. + +## Snapshot from the CLI + +```bash +framevitals snapshot production.parquet \ + --mode quick \ + --output production.snapshot.json +``` + +Snapshot generation always disables filesystem analysis artifacts. The requested +snapshot file is the only explicit monitoring artifact written by the command. + +Use JSON on stdout when integrating with another tool: + +```bash +framevitals snapshot production.parquet \ + --mode quick \ + --format json \ + --output production.snapshot.json +``` + +## Compare two snapshots + +```bash +framevitals compare-snapshots \ + baseline.snapshot.json \ + production.snapshot.json +``` + +The comparison reports: + +- whether the state fingerprint changed; +- added or removed columns; +- dtype changes; +- per-column missingness changes; +- health-score delta; +- ML-readiness delta; +- new findings; +- resolved findings. + +For machine-readable output: + +```bash +framevitals compare-snapshots \ + baseline.snapshot.json \ + production.snapshot.json \ + --format json \ + --output snapshot-diff.json +``` + +## Fail CI when state changes + +Snapshot change is not automatically treated as failure because many changes are +legitimate. When a pipeline explicitly wants fingerprint stability, opt in: + +```bash +framevitals compare-snapshots \ + baseline.snapshot.json \ + production.snapshot.json \ + --fail-on-change +``` + +The command returns exit code `1` when the fingerprints differ and `0` when they are +equal. + +For data-quality enforcement, prefer `framevitals gate` when you want semantic +contract/drift thresholds rather than a strict "anything changed" rule. + +## Persistent history in Python + +For repeated local monitoring, `SnapshotHistory` stores compact snapshots under one +directory: + +```python +history = fv.SnapshotHistory(".framevitals/history") +history.add(report.snapshot(), label="production") + +latest = history.latest() +previous = history.previous() +change = history.compare_latest() +``` + +The default `.framevitals/` runtime directory is ignored by Git. + +## Snapshot compatibility + +Snapshots carry their own schema version. `fv.load_snapshot()` validates that version +before returning an `AnalysisSnapshot`, so incompatible monitoring state fails loudly +instead of being compared with unknown semantics. + +During the `0.x` series, snapshot/result schemas may still evolve. Breaking changes +must be recorded in the changelog and release notes before promotion to `main`. diff --git a/docs/quality-gates.md b/docs/quality-gates.md new file mode 100644 index 0000000..7a778dd --- /dev/null +++ b/docs/quality-gates.md @@ -0,0 +1,164 @@ +# Quality gates and custom checks + +FrameVitals quality gates combine the checks a pipeline actually needs into one +`pass` / `warn` / `fail` verdict. + +## Built-in gate families + +A gate can run any combination of: + +- exact contract validation; +- reference-vs-current drift; +- exact user-defined Python checks. + +```python +result = fv.gate( + current, + reference=reference, + contract=contract, + custom_checks=[positive_revenue], +) + +print(result.status) +print(result.passed) +print(result.reasons) +``` + +At least one family must be selected. + +## Contracts + +Infer a reusable contract from a trusted reference dataset: + +```python +contract = fv.infer_contract(reference) +validation = fv.validate(current, contract) +``` + +Current contracts can encode: + +- required and optional columns; +- broad dtype expectations; +- nullability and tolerated null rates; +- numeric bounds with configurable tolerance; +- low-cardinality allowed values; +- uniqueness hints. + +Contract validation remains exact. If the input is Arrow-, relation-, or file-backed, +FrameVitals may materialize the complete dataset to pandas rather than weakening the +contract to a sample. + +## Custom Python invariants + +```python +@fv.check( + "positive revenue", + severity="error", + description="Revenue cannot be negative.", +) +def positive_revenue(df): + minimum = float(df["revenue"].min()) + return { + "passed": minimum >= 0, + "message": "Negative revenue found." if minimum < 0 else "Revenue is valid.", + "details": {"minimum": minimum}, + } +``` + +Run checks independently: + +```python +result = fv.run_checks(current, [positive_revenue]) +print(result.status) +print(result.findings) +``` + +Or include them in a gate: + +```python +result = fv.gate(current, custom_checks=[positive_revenue]) +``` + +Every custom check receives an isolated DataFrame copy so one check cannot mutate the +input seen by later checks. Exceptions raised by provider code become structured error +results instead of aborting the entire check collection. + +### Severity + +Use `severity="warning"` for soft expectations and `severity="error"` for hard +invariants. A warning-only custom-check result has status `warn` but remains passing; +an error failure produces status `fail`. + +## Third-party check plugins + +External packages can expose reusable checks with normal Python package entry points: + +```toml +[project.entry-points."framevitals.checks"] +positive_revenue = "acme_data_checks:positive_revenue" +``` + +FrameVitals **does not automatically import installed plugins**. Applications opt in: + +```python +checks = fv.discover_checks() +result = fv.gate(current, custom_checks=checks) +``` + +This is an intentional trust boundary: loading an entry point executes code supplied +by the provider package. + +Discovery rejects duplicate public check names and surfaces broken providers rather +than silently changing gate behavior. + +## Drift thresholds + +```python +result = fv.gate( + current, + reference=reference, + drift_warn_on="moderate", + drift_fail_on="severe", +) +``` + +Supported severities are `stable`, `minor`, `moderate`, and `severe`. + +## CLI + +```bash +framevitals gate production.parquet \ + --reference training.parquet \ + --contract contract.json \ + --drift-warn-on moderate \ + --drift-fail-on severe \ + --output framevitals-gate.json +``` + +The CLI exits `0` for pass/warn and `1` for fail. + +## GitHub Action + +The repository includes a reusable composite action: + +```yaml +- uses: parthdongre/FrameVitals@v0.1.0 + id: framevitals + with: + current: production.parquet + reference: training.parquet + contract: contract.json + output: framevitals-gate.json +``` + +For real CI pipelines, pin a released tag or commit SHA instead of a moving branch. +The action exposes `status`, `passed`, and `result-path` outputs. + +## Execution transparency + +The gate's top-level `execution.full_materialization` becomes true if any selected +family required full materialization. Family-specific execution blocks remain nested +under `validation`, `drift`, and `custom` so automation can distinguish why a gate +materialized or sampled data. + +See [Execution provenance](execution-provenance.md) for the shared schema. diff --git a/docs/result-objects.md b/docs/result-objects.md new file mode 100644 index 0000000..c23b6f2 --- /dev/null +++ b/docs/result-objects.md @@ -0,0 +1,169 @@ +# Result objects + +FrameVitals public workflows return **dict-compatible result objects** during the +`0.x` series. This keeps existing mapping/JSON code working while adding discoverable +helpers for notebooks, applications, reports, and CI. + +## Why dict-compatible objects? + +A FrameVitals result should be easy to: + +- index like a normal dictionary; +- serialize to JSON; +- pass into existing application code; +- inspect interactively with named properties and helper methods. + +The `0.x` series therefore favors additive wrappers over a hard break to dataclasses +or validation models. + +## Full analysis + +`fv.analyze(...)` returns `AnalysisResult`. + +```python +report = fv.analyze(data) + +report.health +report.ml_readiness +report.findings +report.recommendations +report.column("age") +report.summary() +report.summary_text() +report.to_json("analysis.json") +report.to_html("analysis.html") +report.snapshot("snapshot.json") +``` + +`AnalysisResult` remains a `dict`, so existing code still works: + +```python +score = report["health"]["overall_score"] +``` + +## Focused diagnostics + +Focused APIs return `DiagnosticResult`: + +- `fv.profile(...)` +- `fv.roles(...)` +- `fv.health(...)` +- `fv.ml_readiness(...)` +- `fv.quality(...)` +- `fv.statistics(...)` +- `fv.anomalies(...)` +- `fv.relationships(...)` +- `fv.target_analysis(...)` + +```python +stats = fv.statistics(data, mode="quick") + +stats.diagnostic # "statistics" +stats.dataset_name +stats.execution +stats.source +stats.available +stats.summary() +stats.summary_text() +stats.to_json("statistics.json") +``` + +The diagnostic label is Python-side metadata and is **not injected into the mapping**. +That means wrapping a focused payload does not mutate its JSON schema: + +```python +assert "diagnostic" not in stats +assert dict(stats) == stats.to_dict() +``` + +`to_dict()` returns a detached deep copy so callers can modify it without mutating the +live result object. + +## Quality operations + +### DriftResult + +`fv.compare(...)` returns `DriftResult` with conveniences such as: + +```python +drift.severity +drift.status +drift.columns +drift.summary_text() +drift.to_json() +``` + +### ValidationResult + +`fv.validate(...)` returns `ValidationResult`: + +```python +validation.valid +validation.status +validation.findings +validation.summary_text() +``` + +### CheckResult + +`fv.run_checks(...)` returns `CheckResult`: + +```python +checks.status +checks.passed +checks.results +checks.findings +checks.summary_text() +``` + +### GateResult + +`fv.gate(...)` returns `GateResult`: + +```python +gate.status +gate.passed +gate.checks_run +gate.reasons +gate.summary_text() +``` + +All four quality result types remain mapping-compatible and expose `to_dict()` / +`to_json()` through the shared quality-result base. + +## Monitoring state + +`AnalysisSnapshot` is a compact dict-compatible monitoring record derived from an +`AnalysisResult`. `SnapshotHistory` manages persisted snapshots without storing raw +input datasets. + +```python +history = fv.SnapshotHistory(".framevitals/history") +history.add(report.snapshot(), label="production") +change = history.compare_latest() +``` + +## Execution metadata + +Focused and quality results increasingly expose the shared execution-provenance +contract under `result.execution` or `result["execution"]`. + +```python +execution = stats.execution +print(execution["execution_schema_version"]) +print(execution["method"]) +print(execution["sampled"]) +print(execution["full_materialization"]) +``` + +See [Execution provenance](execution-provenance.md) for the schema and exactness rules. + +## Compatibility policy during 0.x + +FrameVitals aims to evolve result ergonomics without needless churn: + +1. result objects remain subclasses of `dict`; +2. helper properties should not silently change serialized payloads; +3. new shared metadata is added deliberately and versioned where appropriate; +4. legacy operation-specific fields remain available during migration; +5. breaking result-schema changes must be documented before release. diff --git a/docs/source-execution.md b/docs/source-execution.md new file mode 100644 index 0000000..ec11600 --- /dev/null +++ b/docs/source-execution.md @@ -0,0 +1,154 @@ +# Source-aware execution + +FrameVitals separates **what the data is stored in** from **what a diagnostic needs to +compute**. Public APIs resolve inputs into a `DatasetSource` before choosing between +batch streaming and pandas materialization. + +## Inspect a source first + +```python +import framevitals as fv + +info = fv.inspect_source(data) +print(info) +``` + +Typical fields include: + +```json +{ + "name": "production.parquet", + "kind": "file", + "format": "parquet", + "rows": 1200000, + "columns": 42, + "size_bytes": 84599210, + "materialized": false, + "supports_projection": true, + "supports_streaming": true +} +``` + +`inspect_source()` does not run health/statistical diagnostics. Some adapters may need +to scan lightweight metadata (for example exact relation row counts), but they do not +materialize the full dataset into pandas simply to describe source capabilities. + +## DatasetSource protocol + +A minimal source provides: + +```python +class DatasetSource(Protocol): + def inspect(self) -> DatasetMetadata: ... + def load(self) -> pandas.DataFrame: ... +``` + +A streaming source additionally provides: + +```python +class StreamingDatasetSource(DatasetSource, Protocol): + def iter_batches( + self, + *, + batch_size: int = 65_536, + columns: Sequence[str] | None = None, + ) -> Iterator[Any]: ... +``` + +This means new storage systems can integrate with FrameVitals without changing every +diagnostic implementation. + +## Current adapters + +### pandas + +A pandas `DataFrame` is already materialized. It remains the reference in-memory +representation for exact/full-row operations. + +### Parquet + +With `framevitals[arrow]`, Parquet supports: + +- exact row/column metadata from file metadata; +- column projection; +- Arrow record batches; +- bounded global row sampling without `ParquetSource.load()`. + +### CSV / TSV + +With Arrow installed, compatible delimited files use incremental Arrow CSV reading. +If Arrow cannot interpret a file, FrameVitals preserves compatibility by falling back +to the existing pandas loader rather than silently returning a different parse. + +### PyArrow Table and RecordBatch + +PyArrow in-memory objects enter the batch path directly. FrameVitals does not first +convert the complete object to pandas for streaming-safe diagnostics. + +### Arrow C Stream / PyCapsule producers + +Table-like objects implementing `__arrow_c_stream__` can be normalized through +PyArrow. This gives interoperable support to producers such as modern dataframe +libraries without hard-coding each library into FrameVitals. + +A raw `RecordBatchReader` is currently rejected because it does not expose the cheap +exact row count required by FrameVitals' current planning/profiling contract. + +### DuckDB relations + +With `framevitals[duckdb]`, a lazy `DuckDBPyRelation` supports: + +- exact row count via a cached aggregate; +- schema inspection; +- projection pushed into the DuckDB relation; +- Arrow `RecordBatchReader` transport; +- pandas materialization only when an exact/full-row API explicitly requires it. + +```python +import duckdb +import framevitals as fv + +con = duckdb.connect() +relation = con.sql("SELECT * FROM read_parquet('events/*.parquet')") + +print(fv.inspect_source(relation)) +report = fv.analyze(relation, mode="quick") +``` + +## Source-aware public APIs + +The source abstraction is shared by: + +- `inspect_source()` +- `profile()` +- `roles()` +- `health()` +- `ml_readiness()` +- `quality()` +- `statistics()` +- `anomalies()` +- `relationships()` +- `target_analysis()` +- `analyze()` +- `plan()` +- `compare()` +- `validate()` +- `run_checks()` +- `gate()` + +The operation still decides whether streaming is semantically valid. For example, +contract validation and arbitrary Python checks deliberately remain exact. + +## Adding a source adapter + +Contributors should preserve these invariants: + +1. `inspect()` should be cheaper/safer than full pandas materialization. +2. `iter_batches()` must respect requested projection where supported. +3. source metadata must report the **true** source shape, not sample shape. +4. bounded algorithms must disclose sampling strategy and row counts. +5. exact operations must not claim streaming merely because the source supports it. +6. optional storage dependencies must remain lazily imported. + +See [Execution provenance](execution-provenance.md) for the metadata contract used to +report these choices. diff --git a/docs/stability.md b/docs/stability.md new file mode 100644 index 0000000..f0b9149 --- /dev/null +++ b/docs/stability.md @@ -0,0 +1,138 @@ +# Stability and compatibility policy + +FrameVitals follows semantic versioning, but the project is currently in the `0.x` +series. The purpose of this policy is to make that maturity level explicit without +using "alpha" as an excuse for arbitrary breakage. + +## Python versions + +The current package metadata supports: + +- Python 3.11 +- Python 3.12 +- Python 3.13 + +The main test matrix exercises all three. Separate smoke workflows exercise public +package/CLI behavior on Windows and macOS in addition to the primary Linux CI. + +## Dependency lower bounds + +Minimum core dependency versions declared in `pyproject.toml` are tested in a dedicated +Python 3.11 lower-bound workflow. If FrameVitals begins relying on behavior unavailable +at an advertised lower bound, the project must either restore compatibility or raise +the declared minimum deliberately. + +Optional capability groups (`arrow`, `duckdb`, `excel`, `plot`, `ml`, `ai`, `web`, +`docs`) remain independently installable. Core import/CLI behavior should not require +an optional dependency merely because the corresponding module exists in the source +tree. + +## Public Python surface + +The package-root workflow APIs are the primary public surface: + +- `inspect_source` +- `analyze` / `plan` +- focused diagnostics such as `profile`, `health`, `statistics`, and `relationships` +- `compare` +- `infer_contract` / `validate` +- `check` / `run_checks` / `discover_checks` +- `gate` +- snapshot/history helpers + +A regression test locks the exported package surface so accidental symbol removal is +caught in CI. + +Modules and helpers not exported at package root may still be intentionally reusable, +but should not be assumed to have the same compatibility promise unless documented. + +## Dict-compatible result objects + +During `0.x`, public result objects remain subclasses of `dict` to preserve existing +mapping and JSON-oriented callers while adding helper methods. + +FrameVitals avoids injecting Python-only wrapper metadata into serialized payloads. +For example, `DiagnosticResult.diagnostic` is object metadata and does not become a +new JSON key merely because a focused result is wrapped. + +## Versioned machine-readable schemas + +Several persisted/automation-facing structures carry explicit schema versions: + +- analysis results: `result_schema_version` +- monitoring snapshots: `snapshot_schema_version` +- execution provenance: `execution_schema_version` +- data contracts: contract `version` + +A schema-version bump should accompany an intentionally incompatible machine-readable +change. New additive fields do not necessarily require a bump when older consumers +can ignore them safely. + +## Execution semantics + +Source-aware execution is part of the public behavior, not an implementation detail. +When a result exposes execution provenance, FrameVitals should not silently relabel: + +- sampled work as exact; +- a sample shape as source shape; +- full pandas materialization as streaming; +- approximate cardinality/duplicate estimates as exact values. + +Performance optimizations, including the optional native Rust core, must preserve +public result semantics. Acceleration is not allowed to create a separate correctness +contract. + +## Extension compatibility + +Third-party check plugins use the `framevitals.checks` Python entry-point group. +Plugin discovery is explicit and never automatic because loading an entry point +executes provider code. + +Custom data sources integrate through the `DatasetSource` / `StreamingDatasetSource` +protocols. Prefer standard Arrow interoperability over a library-specific dependency +when the protocol preserves the required metadata and execution semantics. + +## What may change during 0.x + +Before `1.0`, minor releases may still revise: + +- scoring thresholds; +- finding codes or wording; +- result structure not protected by an explicit schema contract; +- source-adapter implementation details; +- experimental analysis modules; +- extension points that are explicitly marked provisional. + +Breaking changes should be documented in `CHANGELOG.md` and release notes before they +reach `main`. Avoid breaking package-root workflow signatures unless the improvement +is substantial enough to justify migration cost. + +## Deprecation direction + +During `0.x`, FrameVitals may occasionally make direct breaking changes when maintaining +two behaviors would create more ambiguity than value. Where practical, prefer a +warning/migration period. + +After `1.0`, the intended policy is stricter: public package-root APIs and versioned +schemas should use documented deprecation periods before removal, except when a change +is required to address a security or correctness issue. + +## CI as the compatibility contract + +The repository intentionally separates compatibility concerns into dedicated lanes: + +- Python 3.11/3.12/3.13 core tests; +- declared minimum dependency tests; +- Arrow streaming tests; +- DuckDB interoperability tests; +- Polars-through-Arrow protocol tests; +- Windows/macOS public smoke tests; +- native Rust bridge checks; +- plugin-provider install/discovery tests; +- package/wheel validation; +- strict documentation build; +- performance guardrails; +- CodeQL security analysis. + +A documented compatibility claim should ideally have a corresponding automated lane +before it is treated as a project promise. diff --git a/examples/check_plugin/README.md b/examples/check_plugin/README.md new file mode 100644 index 0000000..d4f4888 --- /dev/null +++ b/examples/check_plugin/README.md @@ -0,0 +1,49 @@ +# FrameVitals check plugin example + +This directory is a minimal installable third-party package that contributes custom +checks to FrameVitals through standard Python entry points. + +It is intentionally separate from the main `framevitals` package so the example tests +the same packaging/discovery boundary that an external project would use. + +## Install beside a FrameVitals checkout + +From the repository root: + +```bash +python -m pip install -e . +python -m pip install -e examples/check_plugin +``` + +Then discover the provider checks explicitly: + +```python +import framevitals as fv + +checks = fv.discover_checks() +for check in checks: + print(check.name, check.severity) +``` + +Use them directly or inside the quality gate: + +```python +result = fv.gate(dataframe, custom_checks=checks) +print(result.summary_text()) +``` + +## Entry-point contract + +The example declares: + +```toml +[project.entry-points."framevitals.checks"] +positive_revenue = "framevitals_example_checks:positive_revenue" +preferred_plan = "framevitals_example_checks:preferred_plan" +``` + +Provider packages may export `DataCheck` objects or compatible DataFrame callables. +FrameVitals does **not** automatically import installed providers; applications opt in +with `fv.discover_checks()` because loading entry points executes provider code. + +See `docs/extending-framevitals.md` for the extension-author guide. diff --git a/examples/check_plugin/pyproject.toml b/examples/check_plugin/pyproject.toml new file mode 100644 index 0000000..a560913 --- /dev/null +++ b/examples/check_plugin/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "framevitals-example-checks" +version = "0.1.0" +description = "Minimal third-party FrameVitals check plugin example." +requires-python = ">=3.11" +dependencies = [ + "framevitals>=0.1,<1", +] + +[project.entry-points."framevitals.checks"] +positive_revenue = "framevitals_example_checks:positive_revenue" +preferred_plan = "framevitals_example_checks:preferred_plan" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/examples/check_plugin/src/framevitals_example_checks/__init__.py b/examples/check_plugin/src/framevitals_example_checks/__init__.py new file mode 100644 index 0000000..e9928e0 --- /dev/null +++ b/examples/check_plugin/src/framevitals_example_checks/__init__.py @@ -0,0 +1,57 @@ +"""Example third-party checks discovered through FrameVitals entry points.""" + +from __future__ import annotations + +import framevitals as fv + + +@fv.check( + "positive revenue", + severity="error", + description="Revenue cannot be negative.", +) +def positive_revenue(dataframe): + if "revenue" not in dataframe.columns: + return { + "passed": False, + "message": "Required revenue column is missing.", + "details": {"required_column": "revenue"}, + } + + minimum = float(dataframe["revenue"].min()) + return { + "passed": minimum >= 0, + "message": ( + "Revenue is non-negative." + if minimum >= 0 + else "Negative revenue records were found." + ), + "details": {"minimum": minimum}, + } + + +@fv.check( + "preferred plan domain", + severity="warning", + description="Plans should use the preferred product vocabulary.", +) +def preferred_plan(dataframe): + if "plan" not in dataframe.columns: + return { + "passed": False, + "message": "Plan column is missing.", + "details": {"required_column": "plan"}, + } + + allowed = {"basic", "pro", "enterprise"} + observed = set(dataframe["plan"].dropna().astype(str)) + unexpected = sorted(observed - allowed) + return { + "passed": not unexpected, + "message": ( + "Plan values use the preferred domain." + if not unexpected + else "Unexpected plan labels were found." + ), + "details": {"unexpected": unexpected}, + } diff --git a/examples/custom_quality_gate.py b/examples/custom_quality_gate.py new file mode 100644 index 0000000..e5ac1c2 --- /dev/null +++ b/examples/custom_quality_gate.py @@ -0,0 +1,60 @@ +"""Example: combine FrameVitals contracts, drift, and domain-specific checks.""" + +from __future__ import annotations + +import pandas as pd + +import framevitals as fv + + +@fv.check( + "positive revenue", + severity="error", + description="Accounting exports must never contain negative recognized revenue.", +) +def positive_revenue(df: pd.DataFrame): + invalid = int((df["revenue"] < 0).sum()) + return { + "passed": invalid == 0, + "message": ( + "Revenue values are non-negative." + if invalid == 0 + else f"Found {invalid} negative revenue record(s)." + ), + "details": {"negative_rows": invalid}, + } + + +@fv.check("preferred latency budget", severity="warning") +def latency_budget(df: pd.DataFrame): + p95 = float(df["latency_ms"].quantile(0.95)) + return { + "passed": p95 <= 250.0, + "message": f"p95 latency is {p95:.1f} ms; preferred maximum is 250 ms.", + "details": {"p95_latency_ms": p95}, + } + + +def main() -> None: + reference = pd.read_parquet("data/training.parquet") + current = pd.read_parquet("data/production.parquet") + + contract = fv.infer_contract(reference) + result = fv.gate( + current, + reference=reference, + contract=contract, + custom_checks=[positive_revenue, latency_budget], + drift_warn_on="moderate", + drift_fail_on="severe", + ) + + print(result.summary_text()) + result.to_json("framevitals-gate.json") + + if not result.passed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/snapshot_history.py b/examples/snapshot_history.py new file mode 100644 index 0000000..564fefb --- /dev/null +++ b/examples/snapshot_history.py @@ -0,0 +1,34 @@ +"""Example: persist compact FrameVitals monitoring state across recurring runs.""" + +from __future__ import annotations + +import framevitals as fv + + +def main() -> None: + report = fv.analyze("data/production.parquet", mode="quick", artifacts=False) + + history = fv.SnapshotHistory(".framevitals/history") + path = history.add(report, label="nightly") + print(f"Stored snapshot: {path}") + + latest = history.latest() + if latest is not None: + print(f"Latest fingerprint: {latest['fingerprint']}") + + if len(history) >= 2: + change = history.compare_latest() + print(f"Health delta: {change['health_delta']}") + print(f"New findings: {change['findings']['new']}") + + for point in history.timeline()[-5:]: + print( + point["created_at"], + point["health_score"], + point["ml_readiness_score"], + point["finding_count"], + ) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..7c247ea --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,39 @@ +site_name: FrameVitals +site_description: Source-aware data health, drift, contracts, and quality gates for tabular pipelines. +repo_name: parthdongre/FrameVitals +repo_url: https://github.com/parthdongre/FrameVitals +edit_uri: edit/dev/docs/ + +strict: true + +theme: + name: material + features: + - navigation.sections + - navigation.top + - content.code.copy + - search.suggest + +nav: + - Home: index.md + - Source-aware execution: source-execution.md + - Execution provenance: execution-provenance.md + - Result objects: result-objects.md + - Quality gates and custom checks: quality-gates.md + - Monitoring snapshots: monitoring.md + - Extending FrameVitals: extending-framevitals.md + - Stability and compatibility: stability.md + +markdown_extensions: + - admonition + - attr_list + - tables + - toc: + permalink: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.superfences + +plugins: + - search diff --git a/pyproject.toml b/pyproject.toml index 5e35283..f661a15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "framevitals" -version = "0.1.0" +version = "0.2.0" description = "Data quality, drift detection, anomaly analysis, and ML-readiness diagnostics for pandas and tabular data." readme = "README.md" requires-python = ">=3.11" @@ -31,21 +31,25 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Information Analysis", + "Typing :: Typed", ] dependencies = [ "pandas>=2.2", "numpy>=1.26", - "matplotlib>=3.8", - "seaborn>=0.13", - "openpyxl>=3.1", - "xlrd>=2.0", "scipy>=1.13", "statsmodels>=0.14", "scikit-learn>=1.5", - "pydantic>=2.7", ] [project.optional-dependencies] +excel = [ + "openpyxl>=3.1", + "xlrd>=2.0", +] +plot = [ + "matplotlib>=3.8", + "seaborn>=0.13", +] ml = [ "xgboost>=2.0", "lightgbm>=4.3", @@ -54,21 +58,40 @@ ml = [ ] ai = [ "ollama>=0.3", + "pydantic>=2.7", ] web = [ "flask>=3.0", "gunicorn>=22.0", "werkzeug>=3.0", ] +arrow = [ + "pyarrow>=25.0,<26", +] +duckdb = [ + "duckdb>=1.4", + "pyarrow>=25.0,<26", +] +docs = [ + "mkdocs", + "mkdocs-material", +] all = [ + "openpyxl>=3.1", + "xlrd>=2.0", + "matplotlib>=3.8", + "seaborn>=0.13", "xgboost>=2.0", "lightgbm>=4.3", "pyod>=2.0", "shap>=0.45", "ollama>=0.3", + "pydantic>=2.7", "flask>=3.0", "gunicorn>=22.0", "werkzeug>=3.0", + "pyarrow>=25.0,<26", + "duckdb>=1.4", ] dev = [ "pytest>=8.2", @@ -77,13 +100,19 @@ dev = [ "build>=1.2", "twine>=5.1", "ruff>=0.6", + "pre-commit>=4.0", + "openpyxl>=3.1", + "xlrd>=2.0", + "matplotlib>=3.8", + "seaborn>=0.13", + "pydantic>=2.7", ] [project.scripts] framevitals = "framevitals.cli:main" [project.urls] -Documentation = "https://github.com/parthdongre/FrameVitals#readme" +Documentation = "https://github.com/parthdongre/FrameVitals/tree/main/docs" Repository = "https://github.com/parthdongre/FrameVitals" Issues = "https://github.com/parthdongre/FrameVitals/issues" Changelog = "https://github.com/parthdongre/FrameVitals/blob/main/CHANGELOG.md" @@ -95,6 +124,13 @@ package-dir = {"" = "src"} where = ["src"] include = ["framevitals*"] +[tool.setuptools.package-data] +framevitals = ["py.typed"] + +[tool.maturin] +python-source = "src" +module-name = "framevitals._native" + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-ra" diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..acd3556 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,957 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow-array" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown", + "libc", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ord" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-schema" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" +dependencies = [ + "bitflags", +] + +[[package]] +name = "arrow-select" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "framevitals-core" +version = "0.2.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "rustc-hash", +] + +[[package]] +name = "framevitals-py" +version = "0.2.0" +dependencies = [ + "framevitals-core", + "pyo3", + "pyo3-arrow", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "numpy" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4" +dependencies = [ + "half", + "libc", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "pyo3-build-config", + "rustc-hash", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" +dependencies = [ + "chrono", + "chrono-tz", + "indexmap", + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-arrow" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d5ddf226a2dbf7607570d0657c2bf6fbe299208368b2914f3dd7e7ba0b57688" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "chrono", + "chrono-tz", + "half", + "indexmap", + "numpy", + "pyo3", + "thiserror", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..77fdd81 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,3 @@ +[workspace] +members = ["framevitals-core", "framevitals-py"] +resolver = "2" diff --git a/rust/framevitals-core/Cargo.toml b/rust/framevitals-core/Cargo.toml new file mode 100644 index 0000000..3a04db4 --- /dev/null +++ b/rust/framevitals-core/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "framevitals-core" +version = "0.2.0" +edition = "2021" +description = "Native streaming kernels for FrameVitals" +license = "MIT" +publish = false + +[lib] +name = "framevitals_core" +path = "src/lib.rs" + +[features] +default = [] +arrow = ["dep:arrow-array", "dep:arrow-schema"] + +[dependencies] +arrow-array = { version = "59.1", optional = true } +arrow-schema = { version = "59.1", optional = true } +rustc-hash = "2.1" diff --git a/rust/framevitals-core/src/arrow_scan.rs b/rust/framevitals-core/src/arrow_scan.rs new file mode 100644 index 0000000..2460068 --- /dev/null +++ b/rust/framevitals-core/src/arrow_scan.rs @@ -0,0 +1,184 @@ +//! Arrow-native batch scanning for FrameVitals. +//! +//! The scanner consumes Arrow `RecordBatch` values incrementally and produces +//! mergeable numeric states. It never materializes a Python object per cell and +//! does not require a complete dataset to reside in memory. + +use std::collections::{BTreeMap, BTreeSet}; + +use arrow_array::{ + Array, Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, UInt32Array, + UInt64Array, +}; +use arrow_schema::DataType; + +use crate::NumericState; + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct BatchNumericStates { + pub rows: u64, + pub states: BTreeMap, + pub skipped_columns: BTreeSet, +} + +impl BatchNumericStates { + /// Merge states from another Arrow batch/partition without retaining either + /// batch's raw observations. + #[must_use] + pub fn merge(mut self, other: Self) -> Self { + self.rows += other.rows; + self.skipped_columns.extend(other.skipped_columns); + + for (name, state) in other.states { + self.states + .entry(name) + .and_modify(|current| *current = current.merge(state)) + .or_insert(state); + } + self + } +} + +macro_rules! scan_primitive { + ($array:expr) => {{ + let array = $array; + let mut state = NumericState::default(); + for index in 0..array.len() { + if array.is_null(index) { + state.observe(None); + } else { + state.observe(Some(array.value(index) as f64)); + } + } + state + }}; +} + +/// Scan the supported primitive numeric columns of an Arrow `RecordBatch`. +/// +/// Unsupported types are reported in `skipped_columns` rather than failing the +/// complete batch, allowing future semantic/text/sketch scanners to process +/// those columns independently. +pub fn scan_record_batch(batch: &RecordBatch) -> BatchNumericStates { + let mut output = BatchNumericStates { + rows: batch.num_rows() as u64, + ..BatchNumericStates::default() + }; + + for (index, field) in batch.schema().fields().iter().enumerate() { + let name = field.name().clone(); + let array = batch.column(index); + + let state = match field.data_type() { + DataType::Float64 => array + .as_any() + .downcast_ref::() + .map(|values| scan_primitive!(values)), + DataType::Float32 => array + .as_any() + .downcast_ref::() + .map(|values| scan_primitive!(values)), + DataType::Int64 => array + .as_any() + .downcast_ref::() + .map(|values| scan_primitive!(values)), + DataType::Int32 => array + .as_any() + .downcast_ref::() + .map(|values| scan_primitive!(values)), + DataType::UInt64 => array + .as_any() + .downcast_ref::() + .map(|values| scan_primitive!(values)), + DataType::UInt32 => array + .as_any() + .downcast_ref::() + .map(|values| scan_primitive!(values)), + _ => None, + }; + + match state { + Some(state) => { + output.states.insert(name, state); + } + None => { + output.skipped_columns.insert(name); + } + } + } + + output +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{ArrayRef, Float64Array, Int64Array, RecordBatch, StringArray}; + + use super::scan_record_batch; + + fn batch(values: Vec>, integers: Vec>) -> RecordBatch { + RecordBatch::try_from_iter(vec![ + ("value", Arc::new(Float64Array::from(values)) as ArrayRef), + ("count", Arc::new(Int64Array::from(integers)) as ArrayRef), + ( + "label", + Arc::new(StringArray::from(vec![Some("a"), None, Some("c")])) as ArrayRef, + ), + ]) + .expect("valid test batch") + } + + #[test] + fn scans_supported_numeric_columns_and_reports_others() { + let batch = batch( + vec![Some(1.0), None, Some(f64::INFINITY)], + vec![Some(10), Some(20), None], + ); + let state = scan_record_batch(&batch); + + assert_eq!(state.rows, 3); + assert_eq!(state.states.len(), 2); + assert!(state.skipped_columns.contains("label")); + + let value = state.states.get("value").unwrap(); + assert_eq!(value.count, 1); + assert_eq!(value.missing, 1); + assert_eq!(value.infinite, 1); + assert_eq!(value.mean, 1.0); + + let count = state.states.get("count").unwrap(); + assert_eq!(count.count, 2); + assert_eq!(count.missing, 1); + assert_eq!(count.mean, 15.0); + } + + #[test] + fn merges_arrow_batches_into_one_dataset_state() { + let left = batch( + vec![Some(1.0), Some(2.0), None], + vec![Some(1), Some(2), Some(3)], + ); + let right = batch( + vec![Some(4.0), Some(8.0), Some(16.0)], + vec![Some(4), Some(5), Some(6)], + ); + + let merged = scan_record_batch(&left).merge(scan_record_batch(&right)); + let value = merged.states.get("value").unwrap(); + let count = merged.states.get("count").unwrap(); + + assert_eq!(merged.rows, 6); + assert_eq!(value.count, 5); + assert_eq!(value.missing, 1); + assert_eq!(value.minimum, Some(1.0)); + assert_eq!(value.maximum, Some(16.0)); + assert!((value.mean - 6.2).abs() < 1e-12); + + assert_eq!(count.count, 6); + assert_eq!(count.minimum, Some(1.0)); + assert_eq!(count.maximum, Some(6.0)); + assert!((count.mean - 3.5).abs() < 1e-12); + } +} diff --git a/rust/framevitals-core/src/categorical_sketches.rs b/rust/framevitals-core/src/categorical_sketches.rs new file mode 100644 index 0000000..9f31f24 --- /dev/null +++ b/rust/framevitals-core/src/categorical_sketches.rs @@ -0,0 +1,289 @@ +//! Mergeable bounded-memory sketches for UTF-8 categorical values. + +use std::collections::BTreeMap; + +use crate::sketches::{mix64, HyperLogLog}; + +const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; +const DEFAULT_HEAVY_HITTER_CAPACITY: usize = 32; +const DEFAULT_LABEL_BYTES: usize = 256; + +/// Incremental stable byte hasher used by Arrow buffer consumers. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StableByteHasher { + hash: u64, + length: u64, +} + +impl StableByteHasher { + #[must_use] + pub fn new() -> Self { + Self { + hash: FNV_OFFSET_BASIS, + length: 0, + } + } + + pub fn update(&mut self, byte: u8) { + self.hash ^= u64::from(byte); + self.hash = self.hash.wrapping_mul(FNV_PRIME); + self.length = self.length.wrapping_add(1); + } + + #[must_use] + pub fn finish(self) -> u64 { + mix64(self.hash ^ mix64(self.length)) + } +} + +impl Default for StableByteHasher { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct StringCandidate { + label: Vec, + count: u64, +} + +/// Misra-Gries style bounded candidate tracker keyed by a stable full-value hash. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StringHeavyHittersSketch { + capacity: usize, + max_label_bytes: usize, + counters: BTreeMap, +} + +impl StringHeavyHittersSketch { + #[must_use] + pub fn new(capacity: usize, max_label_bytes: usize) -> Self { + assert!(capacity > 0, "heavy-hitter capacity must be positive"); + assert!(max_label_bytes > 0, "max_label_bytes must be positive"); + Self { + capacity, + max_label_bytes, + counters: BTreeMap::new(), + } + } + + pub fn observe_hashed(&mut self, hash: u64, label: &[u8], weight: u64) { + if weight == 0 { + return; + } + if let Some(candidate) = self.counters.get_mut(&hash) { + candidate.count = candidate.count.saturating_add(weight); + return; + } + if self.counters.len() < self.capacity { + self.counters.insert( + hash, + StringCandidate { + label: label[..label.len().min(self.max_label_bytes)].to_vec(), + count: weight, + }, + ); + return; + } + + let minimum = self + .counters + .values() + .map(|candidate| candidate.count) + .min() + .unwrap_or(0); + let decrement = minimum.min(weight); + self.counters.retain(|_, candidate| { + candidate.count -= decrement; + candidate.count > 0 + }); + + let remaining = weight - decrement; + if remaining > 0 { + self.counters.insert( + hash, + StringCandidate { + label: label[..label.len().min(self.max_label_bytes)].to_vec(), + count: remaining, + }, + ); + } + } + + #[must_use] + pub fn merge(mut self, other: Self) -> Self { + assert_eq!( + self.capacity, other.capacity, + "heavy-hitter capacity mismatch" + ); + assert_eq!( + self.max_label_bytes, other.max_label_bytes, + "heavy-hitter label limit mismatch" + ); + for (hash, candidate) in other.counters { + self.observe_hashed(hash, &candidate.label, candidate.count); + } + self + } + + #[must_use] + pub fn candidates(&self) -> Vec<(String, u64)> { + let mut values: Vec<(String, u64)> = self + .counters + .values() + .map(|candidate| { + ( + String::from_utf8_lossy(&candidate.label).into_owned(), + candidate.count, + ) + }) + .collect(); + values.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0))); + values + } + + #[must_use] + pub fn capacity(&self) -> usize { + self.capacity + } + + #[must_use] + pub fn max_label_bytes(&self) -> usize { + self.max_label_bytes + } +} + +impl Default for StringHeavyHittersSketch { + fn default() -> Self { + Self::new(DEFAULT_HEAVY_HITTER_CAPACITY, DEFAULT_LABEL_BYTES) + } +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct CategoricalSketchState { + pub count: u64, + pub missing: u64, + pub cardinality: HyperLogLog, + pub heavy_hitters: StringHeavyHittersSketch, +} + +impl CategoricalSketchState { + pub fn observe_missing(&mut self) { + self.missing = self.missing.saturating_add(1); + } + + pub fn observe_hashed(&mut self, hash: u64, label: &[u8]) { + self.count = self.count.saturating_add(1); + self.cardinality.observe_hash(hash); + self.heavy_hitters.observe_hashed(hash, label, 1); + } + + pub fn observe_bytes(&mut self, value: Option<&[u8]>) { + let Some(value) = value else { + self.observe_missing(); + return; + }; + let mut hasher = StableByteHasher::new(); + for &byte in value { + hasher.update(byte); + } + self.observe_hashed(hasher.finish(), value); + } + + #[must_use] + pub fn merge(self, other: Self) -> Self { + Self { + count: self.count.saturating_add(other.count), + missing: self.missing.saturating_add(other.missing), + cardinality: self.cardinality.merge(other.cardinality), + heavy_hitters: self.heavy_hitters.merge(other.heavy_hitters), + } + } +} + +#[cfg(test)] +mod tests { + use super::{CategoricalSketchState, StableByteHasher, StringHeavyHittersSketch}; + + #[test] + fn stable_hash_is_incremental_and_deterministic() { + let mut left = StableByteHasher::new(); + for byte in b"framevitals" { + left.update(*byte); + } + let mut right = StableByteHasher::new(); + for chunk in [b"frame".as_slice(), b"vitals".as_slice()] { + for byte in chunk { + right.update(*byte); + } + } + assert_eq!(left.finish(), right.finish()); + } + + #[test] + fn categorical_state_tracks_cardinality_missing_and_heavy_hitters() { + let mut state = CategoricalSketchState::default(); + for _ in 0..100 { + state.observe_bytes(Some(b"pune")); + } + for _ in 0..40 { + state.observe_bytes(Some(b"mumbai")); + } + state.observe_bytes(Some(b"nagpur")); + state.observe_bytes(None); + + assert_eq!(state.count, 141); + assert_eq!(state.missing, 1); + assert!((state.cardinality.estimate() - 3.0).abs() < 1.0); + assert_eq!( + state + .heavy_hitters + .candidates() + .first() + .map(|item| item.0.as_str()), + Some("pune") + ); + } + + #[test] + fn categorical_states_merge_without_raw_strings() { + let mut left = CategoricalSketchState::default(); + let mut right = CategoricalSketchState::default(); + for _ in 0..60 { + left.observe_bytes(Some(b"alpha")); + } + for _ in 0..80 { + right.observe_bytes(Some(b"alpha")); + } + left.observe_bytes(Some(b"left")); + right.observe_bytes(Some(b"right")); + right.observe_missing(); + + let merged = left.merge(right); + assert_eq!(merged.count, 142); + assert_eq!(merged.missing, 1); + assert_eq!( + merged + .heavy_hitters + .candidates() + .first() + .map(|item| item.0.as_str()), + Some("alpha") + ); + } + + #[test] + fn labels_are_bounded_even_when_values_are_long() { + let mut sketch = StringHeavyHittersSketch::new(4, 8); + let value = b"abcdefghijklmnopqrstuvwxyz"; + let mut hasher = StableByteHasher::new(); + for byte in value { + hasher.update(*byte); + } + sketch.observe_hashed(hasher.finish(), value, 5); + assert_eq!(sketch.candidates()[0].0, "abcdefgh"); + assert_eq!(sketch.max_label_bytes(), 8); + } +} diff --git a/rust/framevitals-core/src/fused_profile.rs b/rust/framevitals-core/src/fused_profile.rs new file mode 100644 index 0000000..dad7c03 --- /dev/null +++ b/rust/framevitals-core/src/fused_profile.rs @@ -0,0 +1,243 @@ +//! Fused Arrow numeric profiling. +//! +//! One pass over each supported numeric Arrow column updates exact moments and +//! the one sketch consumed by the streaming profile: mergeable log quantiles. +//! Richer HLL/heavy-hitter/reservoir sketches remain available through the +//! standalone numeric profiling APIs, but the full-stream dataframe profiler +//! avoids paying for statistics it never reads. + +use std::collections::{BTreeMap, BTreeSet}; + +use arrow_array::{ + Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, RecordBatch, + UInt16Array, UInt32Array, UInt64Array, UInt8Array, +}; +use arrow_schema::DataType; + +use crate::sketches::LogQuantileSketch; +use crate::NumericState; + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct NumericProfileState { + pub moments: NumericState, + pub quantiles: LogQuantileSketch, +} + +impl NumericProfileState { + pub fn observe(&mut self, value: Option) { + self.moments.observe(value); + if let Some(value) = value { + if value.is_finite() { + self.quantiles.observe(value); + } + } + } + + #[must_use] + pub fn merge(self, other: Self) -> Self { + Self { + moments: self.moments.merge(other.moments), + quantiles: self.quantiles.merge(other.quantiles), + } + } +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct BatchProfileState { + pub rows: u64, + pub profiles: BTreeMap, + pub skipped_columns: BTreeSet, +} + +impl BatchProfileState { + #[must_use] + pub fn merge(mut self, other: Self) -> Self { + self.rows += other.rows; + self.skipped_columns.extend(other.skipped_columns); + for (name, profile) in other.profiles { + self.profiles + .entry(name) + .and_modify(|current| *current = current.clone().merge(profile.clone())) + .or_insert(profile); + } + self + } +} + +macro_rules! scan_profile_primitive { + ($array:expr) => {{ + let array = $array; + let mut profile = NumericProfileState::default(); + for index in 0..array.len() { + if array.is_null(index) { + profile.observe(None); + } else { + profile.observe(Some(array.value(index) as f64)); + } + } + profile + }}; +} + +/// Scan supported primitive numeric columns in one fused pass. +/// +/// ``partition_id`` is retained in the public signature for compatibility with +/// earlier fused-profile callers. Moments and log quantiles are deterministic +/// without partition-specific state. +pub fn profile_record_batch(batch: &RecordBatch, _partition_id: u64) -> BatchProfileState { + let mut output = BatchProfileState { + rows: batch.num_rows() as u64, + ..BatchProfileState::default() + }; + + for (index, field) in batch.schema().fields().iter().enumerate() { + let name = field.name().clone(); + let array = batch.column(index); + + let profile = match field.data_type() { + DataType::Float64 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::Float32 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::Int64 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::Int32 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::Int16 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::Int8 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::UInt64 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::UInt32 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::UInt16 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + DataType::UInt8 => array + .as_any() + .downcast_ref::() + .map(|values| scan_profile_primitive!(values)), + _ => None, + }; + + match profile { + Some(profile) => { + output.profiles.insert(name, profile); + } + None => { + output.skipped_columns.insert(name); + } + } + } + + output +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{ + ArrayRef, Float64Array, Int16Array, Int64Array, RecordBatch, StringArray, UInt8Array, + }; + + use super::profile_record_batch; + + fn batch(values: Vec>, counts: Vec>) -> RecordBatch { + let rows = values.len(); + assert_eq!(counts.len(), rows); + let labels: Vec> = (0..rows) + .map(|index| if index % 2 == 0 { Some("x") } else { None }) + .collect(); + RecordBatch::try_from_iter(vec![ + ("value", Arc::new(Float64Array::from(values)) as ArrayRef), + ("count", Arc::new(Int64Array::from(counts)) as ArrayRef), + ("label", Arc::new(StringArray::from(labels)) as ArrayRef), + ]) + .expect("valid batch") + } + + #[test] + fn fused_profile_updates_moments_and_quantiles_together() { + let batch = batch( + vec![Some(1.0), Some(2.0), None, Some(2.0), Some(f64::INFINITY)], + vec![Some(1), Some(2), Some(3), Some(4), Some(5)], + ); + let state = profile_record_batch(&batch, 7); + let value = state.profiles.get("value").unwrap(); + + assert_eq!(state.rows, 5); + assert_eq!(value.moments.count, 3); + assert_eq!(value.moments.missing, 1); + assert_eq!(value.moments.infinite, 1); + assert!((value.moments.mean - 5.0 / 3.0).abs() < 1e-12); + assert_eq!(value.quantiles.count(), 3); + assert!(value.quantiles.quantile(0.5).is_some()); + assert!(state.skipped_columns.contains("label")); + } + + #[test] + fn fused_profile_supports_compact_integer_arrays() { + let batch = RecordBatch::try_from_iter(vec![ + ( + "small_signed", + Arc::new(Int16Array::from(vec![Some(-2), Some(4), None, Some(8)])) as ArrayRef, + ), + ( + "small_unsigned", + Arc::new(UInt8Array::from(vec![Some(1), Some(2), Some(3), Some(4)])) as ArrayRef, + ), + ]) + .expect("valid compact integer batch"); + + let state = profile_record_batch(&batch, 1); + let signed = state.profiles.get("small_signed").unwrap(); + let unsigned = state.profiles.get("small_unsigned").unwrap(); + + assert_eq!(signed.moments.count, 3); + assert_eq!(signed.moments.missing, 1); + assert_eq!(signed.moments.minimum, Some(-2.0)); + assert_eq!(signed.moments.maximum, Some(8.0)); + assert_eq!(unsigned.moments.count, 4); + assert_eq!(unsigned.moments.mean, 2.5); + } + + #[test] + fn fused_partition_merge_preserves_profile_semantics() { + let left = batch( + (0..1_000).map(|value| Some(value as f64)).collect(), + (0..1_000).map(|value| Some(value as i64)).collect(), + ); + let right = batch( + (1_000..2_000).map(|value| Some(value as f64)).collect(), + (1_000..2_000).map(|value| Some(value as i64)).collect(), + ); + + let merged = profile_record_batch(&left, 1).merge(profile_record_batch(&right, 2)); + let value = merged.profiles.get("value").unwrap(); + + assert_eq!(merged.rows, 2_000); + assert_eq!(value.moments.count, 2_000); + assert!((value.moments.mean - 999.5).abs() < 1e-12); + let median = value.quantiles.quantile(0.5).unwrap(); + assert!(median > 900.0 && median < 1_100.0); + } +} diff --git a/rust/framevitals-core/src/lib.rs b/rust/framevitals-core/src/lib.rs new file mode 100644 index 0000000..a3b3bb3 --- /dev/null +++ b/rust/framevitals-core/src/lib.rs @@ -0,0 +1,316 @@ +//! Native streaming primitives for FrameVitals. +//! +//! This crate intentionally starts with small, well-tested kernels whose +//! semantics match the Python reference implementation. Higher-level Arrow, +//! sketch, graph, and Python bindings can build on these primitives without +//! coupling the analysis engine to pandas. + +#[cfg(feature = "arrow")] +pub mod arrow_scan; +pub mod categorical_sketches; +#[cfg(feature = "arrow")] +pub mod fused_profile; +pub mod sketches; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct NumericState { + pub count: u64, + pub missing: u64, + pub infinite: u64, + pub mean: f64, + pub m2: f64, + pub m3: f64, + pub m4: f64, + pub minimum: Option, + pub maximum: Option, +} + +impl Default for NumericState { + fn default() -> Self { + Self { + count: 0, + missing: 0, + infinite: 0, + mean: 0.0, + m2: 0.0, + m3: 0.0, + m4: 0.0, + minimum: None, + maximum: None, + } + } +} + +impl NumericState { + /// Observe an optional numeric value using stable online central moments + /// through fourth order. + /// + /// `None` and NaN are counted as missing. Positive/negative infinity are + /// counted separately and excluded from finite moments, matching the + /// Python `NumericColumnState` semantics. + pub fn observe(&mut self, value: Option) { + let Some(value) = value else { + self.missing += 1; + return; + }; + + if value.is_nan() { + self.missing += 1; + return; + } + if !value.is_finite() { + self.infinite += 1; + return; + } + + let previous_count = self.count; + self.count += 1; + + let count = self.count as f64; + let previous_count_f = previous_count as f64; + let delta = value - self.mean; + let delta_n = delta / count; + let delta_n2 = delta_n * delta_n; + let term1 = delta * delta_n * previous_count_f; + + // M4/M3 depend on the previous lower-order moments, so update from the + // highest order down before advancing M2 and the mean. + self.m4 += term1 * delta_n2 * (count * count - 3.0 * count + 3.0) + + 6.0 * delta_n2 * self.m2 + - 4.0 * delta_n * self.m3; + self.m3 += term1 * delta_n * (count - 2.0) - 3.0 * delta_n * self.m2; + self.m2 += term1; + self.mean += delta_n; + + self.minimum = Some(match self.minimum { + Some(current) => current.min(value), + None => value, + }); + self.maximum = Some(match self.maximum { + Some(current) => current.max(value), + None => value, + }); + } + + /// Scan a slice into one compact state without retaining observations. + pub fn from_values(values: &[Option]) -> Self { + let mut state = Self::default(); + for &value in values { + state.observe(value); + } + state + } + + /// Merge independently computed partition states through fourth central + /// moment. No raw observations are required. + #[must_use] + pub fn merge(self, other: Self) -> Self { + let missing = self.missing + other.missing; + let infinite = self.infinite + other.infinite; + + if self.count == 0 { + return Self { + missing, + infinite, + ..other + }; + } + if other.count == 0 { + return Self { + missing, + infinite, + ..self + }; + } + + let total = self.count + other.count; + let delta = other.mean - self.mean; + let total_f = total as f64; + let left_f = self.count as f64; + let right_f = other.count as f64; + let delta2 = delta * delta; + let delta3 = delta2 * delta; + let delta4 = delta2 * delta2; + let total2 = total_f * total_f; + let total3 = total2 * total_f; + + let mean = self.mean + delta * right_f / total_f; + let m2 = self.m2 + other.m2 + delta2 * left_f * right_f / total_f; + let m3 = self.m3 + + other.m3 + + delta3 * left_f * right_f * (left_f - right_f) / total2 + + 3.0 * delta * (left_f * other.m2 - right_f * self.m2) / total_f; + let m4 = self.m4 + + other.m4 + + delta4 * left_f * right_f * (left_f * left_f - left_f * right_f + right_f * right_f) + / total3 + + 6.0 * delta2 * (left_f * left_f * other.m2 + right_f * right_f * self.m2) / total2 + + 4.0 * delta * (left_f * other.m3 - right_f * self.m3) / total_f; + + Self { + count: total, + missing, + infinite, + mean, + m2, + m3, + m4, + minimum: match (self.minimum, other.minimum) { + (Some(left), Some(right)) => Some(left.min(right)), + (left @ Some(_), None) => left, + (None, right @ Some(_)) => right, + (None, None) => None, + }, + maximum: match (self.maximum, other.maximum) { + (Some(left), Some(right)) => Some(left.max(right)), + (left @ Some(_), None) => left, + (None, right @ Some(_)) => right, + (None, None) => None, + }, + } + } + + #[must_use] + pub fn variance(&self) -> Option { + if self.count < 2 { + None + } else { + Some(self.m2 / (self.count - 1) as f64) + } + } + + #[must_use] + pub fn standard_deviation(&self) -> Option { + self.variance().map(f64::sqrt) + } + + /// Bias-corrected Fisher-Pearson sample skewness, matching pandas/SciPy + /// `bias=False` semantics used by FrameVitals' deep-statistics report. + #[must_use] + pub fn skewness(&self) -> Option { + if self.count < 3 || self.m2 <= 0.0 { + return None; + } + let n = self.count as f64; + let population_skew = n.sqrt() * self.m3 / self.m2.powf(1.5); + Some((n * (n - 1.0)).sqrt() / (n - 2.0) * population_skew) + } + + /// Bias-corrected Fisher excess kurtosis, matching pandas/SciPy + /// `fisher=True, bias=False` semantics. + #[must_use] + pub fn excess_kurtosis(&self) -> Option { + if self.count < 4 || self.m2 <= 0.0 { + return None; + } + let n = self.count as f64; + let population_excess = n * self.m4 / (self.m2 * self.m2) - 3.0; + Some((n - 1.0) / ((n - 2.0) * (n - 3.0)) * ((n + 1.0) * population_excess + 6.0)) + } +} + +#[cfg(test)] +mod tests { + use super::NumericState; + + fn assert_close(left: f64, right: f64) { + let scale = left.abs().max(right.abs()).max(1.0); + assert!((left - right).abs() <= 1e-10 * scale, "{left} != {right}"); + } + + #[test] + fn scan_tracks_missing_infinite_and_moments() { + let values = [ + Some(1.0), + Some(2.0), + None, + Some(f64::NAN), + Some(f64::INFINITY), + Some(4.0), + ]; + let state = NumericState::from_values(&values); + + assert_eq!(state.count, 3); + assert_eq!(state.missing, 2); + assert_eq!(state.infinite, 1); + assert_eq!(state.minimum, Some(1.0)); + assert_eq!(state.maximum, Some(4.0)); + assert_close(state.mean, 7.0 / 3.0); + assert_close(state.variance().unwrap(), 7.0 / 3.0); + assert!(state.skewness().is_some()); + assert!(state.excess_kurtosis().is_none()); + } + + #[test] + fn partition_merge_matches_single_pass_through_fourth_moment() { + let values: Vec> = (0..10_000) + .map(|value| { + if value % 97 == 0 { + None + } else { + let x = value as f64 * 0.025 - 100.0; + Some(x * x.signum() + (value % 11) as f64) + } + }) + .collect(); + + let full = NumericState::from_values(&values); + let left = NumericState::from_values(&values[..3_333]); + let middle = NumericState::from_values(&values[3_333..7_777]); + let right = NumericState::from_values(&values[7_777..]); + let merged = left.merge(middle).merge(right); + + assert_eq!(merged.count, full.count); + assert_eq!(merged.missing, full.missing); + assert_eq!(merged.infinite, full.infinite); + assert_eq!(merged.minimum, full.minimum); + assert_eq!(merged.maximum, full.maximum); + assert_close(merged.mean, full.mean); + assert_close(merged.m2, full.m2); + assert_close(merged.m3, full.m3); + assert_close(merged.m4, full.m4); + assert_close(merged.variance().unwrap(), full.variance().unwrap()); + assert_close(merged.skewness().unwrap(), full.skewness().unwrap()); + assert_close( + merged.excess_kurtosis().unwrap(), + full.excess_kurtosis().unwrap(), + ); + } + + #[test] + fn known_shape_statistics_match_reference_values() { + let state = NumericState::from_values(&[ + Some(1.0), + Some(2.0), + Some(2.0), + Some(3.0), + Some(9.0), + Some(12.0), + ]); + + assert_close(state.skewness().unwrap(), 1.069_287_452_144_894_3); + assert_close(state.excess_kurtosis().unwrap(), -0.796_319_305_259_674_9); + } + + #[test] + fn merge_preserves_nonfinite_counts_for_empty_partition() { + let finite = NumericState::from_values(&[Some(1.0), Some(2.0)]); + let nonfinite = NumericState::from_values(&[None, Some(f64::NEG_INFINITY)]); + let merged = finite.merge(nonfinite); + + assert_eq!(merged.count, 2); + assert_eq!(merged.missing, 1); + assert_eq!(merged.infinite, 1); + assert_eq!(merged.minimum, Some(1.0)); + assert_eq!(merged.maximum, Some(2.0)); + } + + #[test] + fn constant_values_have_zero_variance_and_undefined_shape() { + let state = NumericState::from_values(&[Some(3.0), Some(3.0), Some(3.0), Some(3.0)]); + assert_eq!(state.variance(), Some(0.0)); + assert_eq!(state.standard_deviation(), Some(0.0)); + assert_eq!(state.skewness(), None); + assert_eq!(state.excess_kurtosis(), None); + } +} diff --git a/rust/framevitals-core/src/sketches.rs b/rust/framevitals-core/src/sketches.rs new file mode 100644 index 0000000..967d382 --- /dev/null +++ b/rust/framevitals-core/src/sketches.rs @@ -0,0 +1,567 @@ +//! Mergeable bounded-memory sketches used by the native FrameVitals profiler. +//! +//! These structures intentionally avoid retaining raw observations. Each sketch +//! can be built per Arrow batch/partition and merged later, which makes the same +//! semantics usable for local streaming, parallel scans, and distributed jobs. + +use std::cmp::Ordering; +use std::collections::BTreeMap; + +use rustc_hash::FxHashMap; + +const DEFAULT_ZERO_THRESHOLD: f64 = 1.0e-12; +type QuantileBins = Vec<(i32, u64)>; + +#[inline] +pub(crate) fn mix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9E37_79B9_7F4A_7C15); + value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + value ^ (value >> 31) +} + +#[inline] +fn canonical_f64_bits(value: f64) -> u64 { + if value == 0.0 { + 0 + } else { + value.to_bits() + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct HyperLogLog { + precision: u8, + registers: Vec, +} + +impl HyperLogLog { + pub fn new(precision: u8) -> Self { + assert!( + (4..=16).contains(&precision), + "HLL precision must be 4..=16" + ); + Self { + precision, + registers: vec![0; 1usize << precision], + } + } + + pub fn observe_f64(&mut self, value: f64) { + if !value.is_finite() { + return; + } + self.observe_hash(mix64(canonical_f64_bits(value))); + } + + pub fn observe_hash(&mut self, hash: u64) { + let index = (hash >> (64 - self.precision)) as usize; + let remainder = hash << self.precision; + let max_rank = 64 - u32::from(self.precision) + 1; + let rank = (remainder.leading_zeros() + 1).min(max_rank) as u8; + self.registers[index] = self.registers[index].max(rank); + } + + #[must_use] + pub fn merge(mut self, other: Self) -> Self { + assert_eq!(self.precision, other.precision, "HLL precision mismatch"); + for (left, right) in self.registers.iter_mut().zip(other.registers) { + *left = (*left).max(right); + } + self + } + + #[must_use] + pub fn estimate(&self) -> f64 { + let m = self.registers.len() as f64; + let alpha = match self.registers.len() { + 16 => 0.673, + 32 => 0.697, + 64 => 0.709, + _ => 0.7213 / (1.0 + 1.079 / m), + }; + let harmonic_sum: f64 = self + .registers + .iter() + .map(|register| 2.0_f64.powi(-i32::from(*register))) + .sum(); + let raw = alpha * m * m / harmonic_sum; + let zero_registers = self.registers.iter().filter(|&&value| value == 0).count(); + + if raw <= 2.5 * m && zero_registers > 0 { + m * (m / zero_registers as f64).ln() + } else { + raw + } + } + + pub fn precision(&self) -> u8 { + self.precision + } + + pub fn bytes_used(&self) -> usize { + self.registers.len() + } +} + +impl Default for HyperLogLog { + fn default() -> Self { + Self::new(12) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct LogQuantileSketch { + relative_accuracy: f64, + log_gamma: f64, + inv_log_gamma: f64, + zero_threshold: f64, + negative: FxHashMap, + positive: FxHashMap, + zero_count: u64, + count: u64, +} + +impl LogQuantileSketch { + pub fn new(relative_accuracy: f64) -> Self { + assert!( + relative_accuracy > 0.0 && relative_accuracy < 1.0, + "relative accuracy must be between 0 and 1" + ); + let gamma = (1.0 + relative_accuracy) / (1.0 - relative_accuracy); + let log_gamma = gamma.ln(); + Self { + relative_accuracy, + log_gamma, + inv_log_gamma: 1.0 / log_gamma, + zero_threshold: DEFAULT_ZERO_THRESHOLD, + negative: FxHashMap::default(), + positive: FxHashMap::default(), + zero_count: 0, + count: 0, + } + } + + #[inline] + fn key(&self, magnitude: f64) -> i32 { + (magnitude.ln() * self.inv_log_gamma).floor() as i32 + } + + #[inline] + fn representative(&self, key: i32) -> f64 { + ((key as f64 + 0.5) * self.log_gamma).exp() + } + + #[inline] + pub fn observe(&mut self, value: f64) { + if !value.is_finite() { + return; + } + self.count += 1; + if value.abs() <= self.zero_threshold { + self.zero_count += 1; + } else if value < 0.0 { + *self.negative.entry(self.key(-value)).or_insert(0) += 1; + } else { + *self.positive.entry(self.key(value)).or_insert(0) += 1; + } + } + + #[must_use] + pub fn merge(mut self, other: Self) -> Self { + assert!( + (self.relative_accuracy - other.relative_accuracy).abs() <= f64::EPSILON, + "quantile sketch accuracy mismatch" + ); + self.count += other.count; + self.zero_count += other.zero_count; + for (key, count) in other.negative { + *self.negative.entry(key).or_insert(0) += count; + } + for (key, count) in other.positive { + *self.positive.entry(key).or_insert(0) += count; + } + self + } + + fn sorted_bins(&self) -> (QuantileBins, QuantileBins) { + let mut negative: QuantileBins = self + .negative + .iter() + .map(|(key, count)| (*key, *count)) + .collect(); + let mut positive: QuantileBins = self + .positive + .iter() + .map(|(key, count)| (*key, *count)) + .collect(); + negative.sort_unstable_by_key(|(key, _)| *key); + positive.sort_unstable_by_key(|(key, _)| *key); + (negative, positive) + } + + fn quantile_from_sorted( + &self, + q: f64, + negative: &[(i32, u64)], + positive: &[(i32, u64)], + ) -> Option { + if self.count == 0 || !(0.0..=1.0).contains(&q) { + return None; + } + let target = (q * (self.count - 1) as f64).floor() as u64; + let mut seen = 0_u64; + + for (key, count) in negative.iter().rev() { + if target < seen + count { + return Some(-self.representative(*key)); + } + seen += count; + } + if target < seen + self.zero_count { + return Some(0.0); + } + seen += self.zero_count; + for (key, count) in positive { + if target < seen + count { + return Some(self.representative(*key)); + } + seen += count; + } + positive + .last() + .map(|(key, _)| self.representative(*key)) + .or_else(|| negative.first().map(|(key, _)| -self.representative(*key))) + .or(Some(0.0)) + } + + pub fn quantile(&self, q: f64) -> Option { + let (negative, positive) = self.sorted_bins(); + self.quantile_from_sorted(q, &negative, &positive) + } + + pub fn quantiles(&self, qs: &[f64]) -> Vec> { + let (negative, positive) = self.sorted_bins(); + qs.iter() + .map(|q| self.quantile_from_sorted(*q, &negative, &positive)) + .collect() + } + + pub fn count(&self) -> u64 { + self.count + } + + pub fn bin_count(&self) -> usize { + self.negative.len() + self.positive.len() + usize::from(self.zero_count > 0) + } + + pub fn relative_accuracy(&self) -> f64 { + self.relative_accuracy + } +} + +impl Default for LogQuantileSketch { + fn default() -> Self { + Self::new(0.01) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct HeavyHittersSketch { + capacity: usize, + counters: BTreeMap, +} + +impl HeavyHittersSketch { + pub fn new(capacity: usize) -> Self { + assert!(capacity > 0, "heavy-hitter capacity must be positive"); + Self { + capacity, + counters: BTreeMap::new(), + } + } + + pub fn observe_f64(&mut self, value: f64) { + if value.is_finite() { + self.observe_weighted(canonical_f64_bits(value), 1); + } + } + + pub fn observe_weighted(&mut self, key: u64, weight: u64) { + if weight == 0 { + return; + } + if let Some(counter) = self.counters.get_mut(&key) { + *counter += weight; + return; + } + if self.counters.len() < self.capacity { + self.counters.insert(key, weight); + return; + } + + let minimum = self.counters.values().copied().min().unwrap_or(0); + let decrement = minimum.min(weight); + self.counters.retain(|_, count| { + *count -= decrement; + *count > 0 + }); + let remaining = weight - decrement; + if remaining > 0 { + self.counters.insert(key, remaining); + } + } + + #[must_use] + pub fn merge(mut self, other: Self) -> Self { + assert_eq!( + self.capacity, other.capacity, + "heavy-hitter capacity mismatch" + ); + for (key, count) in other.counters { + self.observe_weighted(key, count); + } + self + } + + pub fn candidates(&self) -> Vec<(f64, u64)> { + let mut values: Vec<(f64, u64)> = self + .counters + .iter() + .map(|(bits, count)| (f64::from_bits(*bits), *count)) + .collect(); + values.sort_by(|left, right| { + right + .1 + .cmp(&left.1) + .then_with(|| left.0.total_cmp(&right.0)) + }); + values + } + + pub fn capacity(&self) -> usize { + self.capacity + } +} + +impl Default for HeavyHittersSketch { + fn default() -> Self { + Self::new(32) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ReservoirEntry { + pub priority: u64, + pub value: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct PriorityReservoir { + capacity: usize, + entries: Vec, +} + +impl PriorityReservoir { + pub fn new(capacity: usize) -> Self { + assert!(capacity > 0, "reservoir capacity must be positive"); + Self { + capacity, + entries: Vec::with_capacity(capacity), + } + } + + pub fn observe(&mut self, value: f64, stream_id: u64, sequence: u64) { + if !value.is_finite() { + return; + } + let value_hash = mix64(canonical_f64_bits(value)); + let priority = mix64(value_hash ^ mix64(stream_id) ^ mix64(sequence)); + let entry = ReservoirEntry { priority, value }; + + if self.entries.len() < self.capacity { + self.entries.push(entry); + return; + } + if let Some((index, largest)) = self + .entries + .iter() + .enumerate() + .max_by_key(|(_, current)| current.priority) + { + if priority < largest.priority { + self.entries[index] = entry; + } + } + } + + #[must_use] + pub fn merge(mut self, other: Self) -> Self { + assert_eq!(self.capacity, other.capacity, "reservoir capacity mismatch"); + self.entries.extend(other.entries); + self.entries.sort_by_key(|entry| entry.priority); + self.entries.truncate(self.capacity); + self + } + + pub fn values(&self) -> Vec { + let mut entries = self.entries.clone(); + entries.sort_by(|left, right| { + left.value + .partial_cmp(&right.value) + .unwrap_or(Ordering::Equal) + .then_with(|| left.priority.cmp(&right.priority)) + }); + entries.into_iter().map(|entry| entry.value).collect() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn capacity(&self) -> usize { + self.capacity + } +} + +impl Default for PriorityReservoir { + fn default() -> Self { + Self::new(256) + } +} + +#[derive(Debug, Clone, PartialEq, Default)] +pub struct NumericSketchState { + pub cardinality: HyperLogLog, + pub quantiles: LogQuantileSketch, + pub heavy_hitters: HeavyHittersSketch, + pub reservoir: PriorityReservoir, +} + +impl NumericSketchState { + pub fn observe(&mut self, value: f64, stream_id: u64, sequence: u64) { + if !value.is_finite() { + return; + } + self.cardinality.observe_f64(value); + self.quantiles.observe(value); + self.heavy_hitters.observe_f64(value); + self.reservoir.observe(value, stream_id, sequence); + } + + #[must_use] + pub fn merge(self, other: Self) -> Self { + Self { + cardinality: self.cardinality.merge(other.cardinality), + quantiles: self.quantiles.merge(other.quantiles), + heavy_hitters: self.heavy_hitters.merge(other.heavy_hitters), + reservoir: self.reservoir.merge(other.reservoir), + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + HeavyHittersSketch, HyperLogLog, LogQuantileSketch, NumericSketchState, PriorityReservoir, + }; + + #[test] + fn hll_estimates_cardinality_and_merges_partitions() { + let mut left = HyperLogLog::new(12); + let mut right = HyperLogLog::new(12); + for value in 0..10_000 { + if value < 5_000 { + left.observe_f64(value as f64); + } else { + right.observe_f64(value as f64); + } + } + let estimate = left.merge(right).estimate(); + assert!((estimate - 10_000.0).abs() / 10_000.0 < 0.08); + } + + #[test] + fn logarithmic_quantiles_are_mergeable_and_bounded() { + let mut left = LogQuantileSketch::new(0.01); + let mut right = LogQuantileSketch::new(0.01); + for value in 1..=10_000 { + if value <= 5_000 { + left.observe(value as f64); + } else { + right.observe(value as f64); + } + } + let merged = left.merge(right); + let median = merged.quantile(0.5).unwrap(); + assert!((median - 5_000.0).abs() / 5_000.0 < 0.03); + assert!(merged.bin_count() < 1_000); + } + + #[test] + fn batched_log_quantiles_match_individual_queries() { + let mut sketch = LogQuantileSketch::new(0.01); + for value in -5_000..=5_000 { + sketch.observe(value as f64); + } + let levels = [0.01, 0.25, 0.5, 0.75, 0.99]; + let batched = sketch.quantiles(&levels); + let individual: Vec<_> = levels.iter().map(|q| sketch.quantile(*q)).collect(); + assert_eq!(batched, individual); + } + + #[test] + fn heavy_hitters_retain_dominant_candidates_after_merge() { + let mut left = HeavyHittersSketch::new(8); + let mut right = HeavyHittersSketch::new(8); + for _ in 0..100 { + left.observe_f64(7.0); + } + for value in 0..50 { + left.observe_f64(value as f64); + right.observe_f64((value + 50) as f64); + } + for _ in 0..80 { + right.observe_f64(7.0); + } + let candidates = left.merge(right).candidates(); + assert_eq!(candidates.first().map(|item| item.0), Some(7.0)); + } + + #[test] + fn priority_reservoir_is_bounded_and_partition_mergeable() { + let mut left = PriorityReservoir::new(32); + let mut right = PriorityReservoir::new(32); + for value in 0..1_000_u64 { + if value < 500 { + left.observe(value as f64, 1, value); + } else { + right.observe(value as f64, 2, value - 500); + } + } + let merged = left.merge(right); + assert_eq!(merged.len(), 32); + assert_eq!(merged.values().len(), 32); + } + + #[test] + fn combined_numeric_sketch_merges_without_raw_values() { + let mut left = NumericSketchState::default(); + let mut right = NumericSketchState::default(); + for value in 0..2_000_u64 { + if value < 1_000 { + left.observe(value as f64, 11, value); + } else { + right.observe(value as f64, 12, value - 1_000); + } + } + let merged = left.merge(right); + assert!(merged.cardinality.estimate() > 1_800.0); + assert!(merged.quantiles.quantile(0.5).unwrap() > 900.0); + assert_eq!(merged.reservoir.len(), 256); + } +} diff --git a/rust/framevitals-py/Cargo.toml b/rust/framevitals-py/Cargo.toml new file mode 100644 index 0000000..3c5325a --- /dev/null +++ b/rust/framevitals-py/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "framevitals-py" +version = "0.2.0" +edition = "2021" +description = "PyO3 bridge for FrameVitals native kernels" +license = "MIT" +publish = false + +[lib] +name = "_native" +path = "src/lib.rs" +crate-type = ["cdylib", "rlib"] + +[dependencies] +framevitals-core = { path = "../framevitals-core", features = ["arrow"] } +pyo3 = { version = "0.29", features = ["abi3-py311"] } +pyo3-arrow = "0.19" diff --git a/rust/framevitals-py/src/lib.rs b/rust/framevitals-py/src/lib.rs new file mode 100644 index 0000000..d8712c4 --- /dev/null +++ b/rust/framevitals-py/src/lib.rs @@ -0,0 +1,319 @@ +//! Python bindings for FrameVitals native kernels. + +mod string_accumulator; + +use framevitals_core::fused_profile::{profile_record_batch, BatchProfileState}; +use framevitals_core::sketches::{LogQuantileSketch, NumericSketchState}; +use framevitals_core::NumericState; +use pyo3::buffer::PyBuffer; +use pyo3::exceptions::PyBufferError; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use pyo3_arrow::PyRecordBatch; +use string_accumulator::StringAccumulator; + +fn exact_state_dict<'py>(py: Python<'py>, state: &NumericState) -> PyResult> { + let payload = PyDict::new(py); + payload.set_item("count", state.count)?; + payload.set_item("missing", state.missing)?; + payload.set_item("infinite", state.infinite)?; + payload.set_item( + "mean", + if state.count > 0 { + Some(state.mean) + } else { + None + }, + )?; + payload.set_item("m2", state.m2)?; + payload.set_item("m3", state.m3)?; + payload.set_item("m4", state.m4)?; + payload.set_item("variance", state.variance())?; + payload.set_item("std", state.standard_deviation())?; + payload.set_item("skewness", state.skewness())?; + payload.set_item("kurtosis", state.excess_kurtosis())?; + payload.set_item("minimum", state.minimum)?; + payload.set_item("maximum", state.maximum)?; + Ok(payload) +} + +fn profile_dict<'py>( + py: Python<'py>, + state: &NumericState, + sketches: &NumericSketchState, + observations: u64, +) -> PyResult> { + let payload = exact_state_dict(py, state)?; + payload.set_item("backend", "rust")?; + payload.set_item("observations", observations)?; + payload.set_item( + "cardinality_estimate", + sketches.cardinality.estimate().round() as u64, + )?; + + let quantiles = PyDict::new(py); + for (name, q) in [ + ("p01", 0.01), + ("p05", 0.05), + ("p25", 0.25), + ("p50", 0.50), + ("p75", 0.75), + ("p95", 0.95), + ("p99", 0.99), + ] { + quantiles.set_item(name, sketches.quantiles.quantile(q))?; + } + quantiles.set_item("relative_accuracy", sketches.quantiles.relative_accuracy())?; + payload.set_item("quantiles", quantiles)?; + payload.set_item("heavy_hitters", sketches.heavy_hitters.candidates())?; + payload.set_item("reservoir", sketches.reservoir.values())?; + Ok(payload) +} + +fn streaming_profile_dict<'py>( + py: Python<'py>, + state: &NumericState, + quantile_sketch: &LogQuantileSketch, + observations: u64, +) -> PyResult> { + let payload = exact_state_dict(py, state)?; + payload.set_item("backend", "rust")?; + payload.set_item("observations", observations)?; + payload.set_item("sketch_policy", "moments_and_log_quantiles")?; + + let quantiles = PyDict::new(py); + for (name, q) in [ + ("p01", 0.01), + ("p05", 0.05), + ("p25", 0.25), + ("p50", 0.50), + ("p75", 0.75), + ("p95", 0.95), + ("p99", 0.99), + ] { + quantiles.set_item(name, quantile_sketch.quantile(q))?; + } + quantiles.set_item("relative_accuracy", quantile_sketch.relative_accuracy())?; + payload.set_item("quantiles", quantiles)?; + Ok(payload) +} + +fn batch_profile_dict<'py>( + py: Python<'py>, + state: &BatchProfileState, +) -> PyResult> { + let payload = PyDict::new(py); + payload.set_item("backend", "rust")?; + payload.set_item("rows", state.rows)?; + payload.set_item("sketch_policy", "moments_and_log_quantiles")?; + + let profiles = PyDict::new(py); + for (name, profile) in &state.profiles { + let observations = + profile.moments.count + profile.moments.missing + profile.moments.infinite; + profiles.set_item( + name, + streaming_profile_dict(py, &profile.moments, &profile.quantiles, observations)?, + )?; + } + payload.set_item("profiles", profiles)?; + payload.set_item( + "skipped_columns", + state.skipped_columns.iter().cloned().collect::>(), + )?; + Ok(payload) +} + +fn checked_buffer<'py>(values: &Bound<'py, PyAny>) -> PyResult> { + let buffer = PyBuffer::::get(values)?; + if buffer.dimensions() != 1 { + return Err(PyBufferError::new_err( + "FrameVitals native numeric kernels require a 1D float64 buffer.", + )); + } + if !buffer.is_c_contiguous() { + return Err(PyBufferError::new_err( + "FrameVitals native numeric kernels require a C-contiguous float64 buffer.", + )); + } + Ok(buffer) +} + +fn update_states( + py: Python<'_>, + values: &Bound<'_, PyAny>, + state: &mut NumericState, + sketches: &mut NumericSketchState, + stream_id: u64, + sequence: &mut u64, +) -> PyResult { + let buffer = checked_buffer(values)?; + let slice = buffer.as_slice(py).ok_or_else(|| { + PyBufferError::new_err( + "FrameVitals could not borrow the supplied float64 buffer as a contiguous slice.", + ) + })?; + + for value in slice { + let value = value.get(); + state.observe(Some(value)); + if value.is_finite() { + sketches.observe(value, stream_id, *sequence); + } + *sequence = sequence.wrapping_add(1); + } + Ok(buffer.item_count() as u64) +} + +#[pyclass] +struct NumericAccumulator { + state: NumericState, + sketches: NumericSketchState, + stream_id: u64, + sequence: u64, + observations: u64, +} + +#[pymethods] +impl NumericAccumulator { + #[new] + #[pyo3(signature = (stream_id = 0))] + fn new(stream_id: u64) -> Self { + Self { + state: NumericState::default(), + sketches: NumericSketchState::default(), + stream_id, + sequence: 0, + observations: 0, + } + } + + fn update_f64(&mut self, py: Python<'_>, values: &Bound<'_, PyAny>) -> PyResult<()> { + let observed = update_states( + py, + values, + &mut self.state, + &mut self.sketches, + self.stream_id, + &mut self.sequence, + )?; + self.observations = self.observations.wrapping_add(observed); + Ok(()) + } + + fn snapshot(&self, py: Python<'_>) -> PyResult> { + Ok(profile_dict(py, &self.state, &self.sketches, self.observations)?.unbind()) + } + + fn reset(&mut self) { + self.state = NumericState::default(); + self.sketches = NumericSketchState::default(); + self.sequence = 0; + self.observations = 0; + } + + #[getter] + fn observations(&self) -> u64 { + self.observations + } +} + +/// Persistent zero-copy Arrow batch profiler. +/// +/// ``pyo3-arrow`` imports ``pyarrow.RecordBatch`` through the Arrow PyCapsule +/// interface. Numeric primitive arrays are scanned directly in Rust without +/// constructing NumPy float64 copies or dispatching once per column in Python. +#[pyclass] +struct ArrowBatchProfileAccumulator { + state: BatchProfileState, + next_partition_id: u64, +} + +#[pymethods] +impl ArrowBatchProfileAccumulator { + #[new] + fn new() -> Self { + Self { + state: BatchProfileState::default(), + next_partition_id: 0, + } + } + + fn update(&mut self, batch: PyRecordBatch) { + let partition = profile_record_batch(batch.as_ref(), self.next_partition_id); + let current = std::mem::take(&mut self.state); + self.state = current.merge(partition); + self.next_partition_id = self.next_partition_id.wrapping_add(1); + } + + fn snapshot(&self, py: Python<'_>) -> PyResult> { + Ok(batch_profile_dict(py, &self.state)?.unbind()) + } + + fn reset(&mut self) { + self.state = BatchProfileState::default(); + self.next_partition_id = 0; + } + + #[getter] + fn rows(&self) -> u64 { + self.state.rows + } +} + +#[pyfunction] +fn numeric_state_f64(py: Python<'_>, values: &Bound<'_, PyAny>) -> PyResult> { + let buffer = checked_buffer(values)?; + let slice = buffer.as_slice(py).ok_or_else(|| { + PyBufferError::new_err( + "FrameVitals could not borrow the supplied float64 buffer as a contiguous slice.", + ) + })?; + + let mut state = NumericState::default(); + for value in slice { + state.observe(Some(value.get())); + } + let payload = exact_state_dict(py, &state)?; + payload.set_item("backend", "rust")?; + payload.set_item("observations", buffer.item_count())?; + Ok(payload.unbind()) +} + +#[pyfunction] +#[pyo3(signature = (values, stream_id = 0))] +fn numeric_profile_f64( + py: Python<'_>, + values: &Bound<'_, PyAny>, + stream_id: u64, +) -> PyResult> { + let mut state = NumericState::default(); + let mut sketches = NumericSketchState::default(); + let mut sequence = 0_u64; + let observations = update_states( + py, + values, + &mut state, + &mut sketches, + stream_id, + &mut sequence, + )?; + Ok(profile_dict(py, &state, &sketches, observations)?.unbind()) +} + +#[pyfunction] +fn backend_info() -> (&'static str, &'static str) { + ("rust", env!("CARGO_PKG_VERSION")) +} + +#[pymodule] +fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(numeric_state_f64, module)?)?; + module.add_function(wrap_pyfunction!(numeric_profile_f64, module)?)?; + module.add_function(wrap_pyfunction!(backend_info, module)?)?; + module.add("__version__", env!("CARGO_PKG_VERSION"))?; + Ok(()) +} diff --git a/rust/framevitals-py/src/string_accumulator.rs b/rust/framevitals-py/src/string_accumulator.rs new file mode 100644 index 0000000..c2699bf --- /dev/null +++ b/rust/framevitals-py/src/string_accumulator.rs @@ -0,0 +1,205 @@ +use framevitals_core::categorical_sketches::{CategoricalSketchState, StableByteHasher}; +use pyo3::buffer::{Element, PyBuffer, ReadOnlyCell}; +use pyo3::exceptions::PyBufferError; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +trait OffsetValue: Element + Copy { + fn as_i64(self) -> i64; +} + +impl OffsetValue for i32 { + fn as_i64(self) -> i64 { + i64::from(self) + } +} + +impl OffsetValue for i64 { + fn as_i64(self) -> i64 { + self + } +} + +fn is_valid(validity: Option<&[ReadOnlyCell]>, index: usize) -> bool { + let Some(validity) = validity else { + return true; + }; + let byte_index = index / 8; + if byte_index >= validity.len() { + return false; + } + let bit = index % 8; + validity[byte_index].get() & (1_u8 << bit) != 0 +} + +fn update_utf8_buffers( + py: Python<'_>, + state: &mut CategoricalSketchState, + data: &Bound<'_, PyAny>, + offsets: &Bound<'_, PyAny>, + length: usize, + validity: Option<&Bound<'_, PyAny>>, + array_offset: usize, +) -> PyResult<()> { + let data_buffer = PyBuffer::::get(data)?; + let offsets_buffer = PyBuffer::::get(offsets)?; + let validity_buffer = validity.map(PyBuffer::::get).transpose()?; + + if !data_buffer.is_c_contiguous() || !offsets_buffer.is_c_contiguous() { + return Err(PyBufferError::new_err( + "FrameVitals string kernels require contiguous Arrow data/offset buffers.", + )); + } + if let Some(buffer) = &validity_buffer { + if !buffer.is_c_contiguous() { + return Err(PyBufferError::new_err( + "FrameVitals string kernels require a contiguous validity bitmap.", + )); + } + } + + let data_slice = data_buffer.as_slice(py).ok_or_else(|| { + PyBufferError::new_err("Could not borrow Arrow UTF8 data as a byte slice.") + })?; + let offsets_slice = offsets_buffer.as_slice(py).ok_or_else(|| { + PyBufferError::new_err("Could not borrow Arrow UTF8 offsets as a typed slice.") + })?; + let validity_slice = validity_buffer + .as_ref() + .and_then(|buffer| buffer.as_slice(py)); + + let required_offsets = array_offset + .checked_add(length) + .and_then(|value| value.checked_add(1)) + .ok_or_else(|| PyBufferError::new_err("Arrow string offset range overflowed."))?; + if required_offsets > offsets_slice.len() { + return Err(PyBufferError::new_err( + "Arrow string offsets are shorter than the requested array range.", + )); + } + + let label_limit = state.heavy_hitters.max_label_bytes(); + for local_index in 0..length { + let logical_index = array_offset + local_index; + if !is_valid(validity_slice, logical_index) { + state.observe_missing(); + continue; + } + + let start = offsets_slice[logical_index].get().as_i64(); + let end = offsets_slice[logical_index + 1].get().as_i64(); + if start < 0 || end < start { + return Err(PyBufferError::new_err( + "Arrow string offsets contain an invalid range.", + )); + } + let start = usize::try_from(start) + .map_err(|_| PyBufferError::new_err("Arrow string offset is out of range."))?; + let end = usize::try_from(end) + .map_err(|_| PyBufferError::new_err("Arrow string offset is out of range."))?; + if end > data_slice.len() { + return Err(PyBufferError::new_err( + "Arrow string offset exceeds the UTF8 data buffer.", + )); + } + + let mut hasher = StableByteHasher::new(); + let mut label = Vec::with_capacity((end - start).min(label_limit)); + for cell in &data_slice[start..end] { + let byte = cell.get(); + hasher.update(byte); + if label.len() < label_limit { + label.push(byte); + } + } + state.observe_hashed(hasher.finish(), &label); + } + Ok(()) +} + +#[pyclass] +pub(crate) struct StringAccumulator { + state: CategoricalSketchState, +} + +#[pymethods] +impl StringAccumulator { + #[new] + fn new() -> Self { + Self { + state: CategoricalSketchState::default(), + } + } + + #[pyo3(signature = (data, offsets, length, validity=None, array_offset=0))] + fn update_utf8( + &mut self, + py: Python<'_>, + data: &Bound<'_, PyAny>, + offsets: &Bound<'_, PyAny>, + length: usize, + validity: Option<&Bound<'_, PyAny>>, + array_offset: usize, + ) -> PyResult<()> { + update_utf8_buffers::( + py, + &mut self.state, + data, + offsets, + length, + validity, + array_offset, + ) + } + + #[pyo3(signature = (data, offsets, length, validity=None, array_offset=0))] + fn update_large_utf8( + &mut self, + py: Python<'_>, + data: &Bound<'_, PyAny>, + offsets: &Bound<'_, PyAny>, + length: usize, + validity: Option<&Bound<'_, PyAny>>, + array_offset: usize, + ) -> PyResult<()> { + update_utf8_buffers::( + py, + &mut self.state, + data, + offsets, + length, + validity, + array_offset, + ) + } + + fn snapshot(&self, py: Python<'_>) -> PyResult> { + let payload = PyDict::new(py); + payload.set_item("backend", "rust")?; + payload.set_item("count", self.state.count)?; + payload.set_item("missing", self.state.missing)?; + payload.set_item("observations", self.state.count + self.state.missing)?; + payload.set_item( + "cardinality_estimate", + self.state.cardinality.estimate().round() as u64, + )?; + payload.set_item("heavy_hitters", self.state.heavy_hitters.candidates())?; + payload.set_item("cardinality_method", "hyperloglog")?; + payload.set_item("heavy_hitter_method", "misra_gries_candidates")?; + payload.set_item("heavy_hitter_count_semantics", "lower_bound")?; + payload.set_item( + "max_retained_label_bytes", + self.state.heavy_hitters.max_label_bytes(), + )?; + Ok(payload.unbind()) + } + + fn reset(&mut self) { + self.state = CategoricalSketchState::default(); + } + + #[getter] + fn observations(&self) -> u64 { + self.state.count + self.state.missing + } +} diff --git a/src/framevitals/__init__.py b/src/framevitals/__init__.py index 1ca3332..4039a1a 100644 --- a/src/framevitals/__init__.py +++ b/src/framevitals/__init__.py @@ -2,42 +2,241 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any -__version__ = "0.1.0" +from framevitals.config import AnalysisConfig, available_modules +from framevitals.planning import AnalysisPlan +from framevitals.quality_results import ( + CheckResult, + DriftResult, + GateResult, + ValidationResult, +) +from framevitals.result import AnalysisResult, ColumnResult, DiagnosticResult +from framevitals.snapshots import ( + AnalysisSnapshot, + SnapshotHistory, + compare_snapshots, + create_snapshot, + load_snapshot, +) +if TYPE_CHECKING: + from framevitals.checks import DataCheck + from framevitals.cleaning_plan import CleaningPlan -def analyze( +__version__ = "0.2.0" + + +def __getattr__(name: str): + """Lazily expose optional/heavier public types.""" + if name == "CleaningPlan": + from framevitals.cleaning_plan import CleaningPlan + + return CleaningPlan + if name == "DataCheck": + from framevitals.checks import DataCheck + + return DataCheck + raise AttributeError(name) + + +def inspect_source(data: Any) -> dict[str, Any]: + """Inspect source metadata and execution capabilities without analysis.""" + from framevitals.sources import inspect_source as _inspect_source + + return _inspect_source(data) + + +def profile(data: Any) -> DiagnosticResult: + """Profile shape, dtypes, missingness, cardinality, and basic summaries.""" + from framevitals.focused import profile as _profile + + return _profile(data) + + +def roles(data: Any) -> DiagnosticResult: + """Infer semantic and structural roles for dataset columns.""" + from framevitals.focused import roles as _roles + + return _roles(data) + + +def health(data: Any) -> DiagnosticResult: + """Calculate only the FrameVitals data-health score.""" + from framevitals.focused import health as _health + + return _health(data) + + +def ml_readiness(data: Any) -> DiagnosticResult: + """Calculate only ML-readiness diagnostics.""" + from framevitals.focused import ml_readiness as _ml_readiness + + return _ml_readiness(data) + + +def quality( data: Any, *, - target: str | None = None, + max_sample_rows: int = 5_000, + max_columns: int = 100, + max_missingness_columns: int = 25, +) -> DiagnosticResult: + """Run practical deterministic data-quality diagnostics only.""" + from framevitals.focused import quality as _quality + + return _quality( + data, + max_sample_rows=max_sample_rows, + max_columns=max_columns, + max_missingness_columns=max_missingness_columns, + ) + + +def statistics( + data: Any, + *, + max_pairs: int = 20, mode: str = "standard", - artifacts: bool = False, -) -> dict[str, Any]: - """Analyze a tabular dataset from a DataFrame or supported file path. +) -> DiagnosticResult: + """Run the deep statistical diagnostics layer with adaptive row budgets.""" + from framevitals.focused import statistics as _statistics + + return _statistics(data, max_pairs=max_pairs, mode=mode) + + +def anomalies( + data: Any, + *, + contamination: float = 0.05, + threshold: float = 0.6, + max_columns: int = 30, + top_k: int = 25, + mode: str = "standard", +) -> DiagnosticResult: + """Run only the bounded FrameVitals tabular anomaly ensemble.""" + from framevitals.focused import anomalies as _anomalies + + return _anomalies( + data, + contamination=contamination, + threshold=threshold, + max_columns=max_columns, + top_k=top_k, + mode=mode, + ) + + +def relationships( + data: Any, + *, + max_sample_rows: int = 512, + projections: int = 64, + min_abs_correlation: float = 0.80, + max_candidate_pairs: int = 250_000, + max_edges_returned: int = 5_000, +) -> DiagnosticResult: + """Discover strong numeric relationships without a dense correlation matrix.""" + from framevitals.focused import relationships as _relationships + + return _relationships( + data, + max_sample_rows=max_sample_rows, + projections=projections, + min_abs_correlation=min_abs_correlation, + max_candidate_pairs=max_candidate_pairs, + max_edges_returned=max_edges_returned, + ) + + +def system_info(*, probe_gpu: bool = True) -> dict[str, Any]: + """Inspect FrameVitals CPU/native/CUDA capabilities without installing anything.""" + from framevitals.acceleration import system_info as _system_info - The implementation is imported lazily so ``import framevitals`` and - ``framevitals --version`` do not initialize the complete analytics stack. - """ - from framevitals.api import analyze as _analyze + return _system_info(probe_gpu=probe_gpu) + + +def target_analysis(data: Any, *, target: str) -> DiagnosticResult: + """Run target quality, leakage, association, and split diagnostics only.""" + from framevitals.focused import target_analysis as _target_analysis + + return _target_analysis(data, target=target) + + +def analyze( + data: Any, + *, + target: str | None = None, + mode: str | None = None, + artifacts: bool | None = None, + workers: int | None = None, + preset: str | None = None, + config: Any = None, + disabled_modules: list[str] | tuple[str, ...] | None = None, +) -> AnalysisResult: + """Analyze a supported tabular source through the canonical dispatcher.""" + from framevitals.analysis_api import analyze as _analyze return _analyze( data, target=target, mode=mode, artifacts=artifacts, + workers=workers, + preset=preset, + config=config, + disabled_modules=disabled_modules, ) +def plan( + data: Any, + *, + target: str | None = None, + mode: str | None = None, + workers: int | None = None, + preset: str | None = None, + config: Any = None, + disabled_modules: list[str] | tuple[str, ...] | None = None, +) -> AnalysisPlan: + """Preview analyses, scale policy, and execution constraints without running them.""" + from framevitals.planning_api import plan as _plan + + return _plan( + data, + target=target, + mode=mode, + workers=workers, + preset=preset, + config=config, + disabled_modules=disabled_modules, + ) + + +def plan_cleaning(data: Any) -> CleaningPlan: + """Infer a conservative cleaning plan without changing the input data.""" + from framevitals.operations import plan_cleaning as _plan_cleaning + + return _plan_cleaning(data) + + +def clean(data: Any, *, plan: Any = None): + """Return an explicitly cleaned copy using a supplied or inferred plan.""" + from framevitals.operations import clean as _clean + + return _clean(data, plan=plan) + + def compare( reference: Any, current: Any, *, columns: list[str] | None = None, max_columns: int = 30, -) -> dict[str, Any]: +) -> DriftResult: """Compare two datasets and return a structured drift report.""" - from framevitals.api import compare as _compare + from framevitals.operations import compare as _compare return _compare( reference, @@ -47,18 +246,101 @@ def compare( ) -def infer_contract(data: Any) -> dict[str, Any]: +def infer_contract( + data: Any, + *, + numeric_tolerance: float = 0.05, + max_categories: int = 20, + null_fraction_tolerance: float = 0.05, + infer_unique: bool = True, + min_unique_rows: int = 20, + allow_extra_columns: bool = False, +) -> dict[str, Any]: """Infer a reusable data contract from a reference dataset.""" - from framevitals.api import infer_contract as _infer_contract + from framevitals.operations import infer_contract as _infer_contract - return _infer_contract(data) + return _infer_contract( + data, + numeric_tolerance=numeric_tolerance, + max_categories=max_categories, + null_fraction_tolerance=null_fraction_tolerance, + infer_unique=infer_unique, + min_unique_rows=min_unique_rows, + allow_extra_columns=allow_extra_columns, + ) -def validate(data: Any, contract: dict[str, Any]) -> dict[str, Any]: +def validate(data: Any, contract: dict[str, Any]) -> ValidationResult: """Validate a dataset against an inferred or explicit data contract.""" - from framevitals.api import validate as _validate + from framevitals.operations import validate as _validate return _validate(data, contract) -__all__ = ["analyze", "compare", "infer_contract", "validate", "__version__"] +def check( + name: str | None = None, + *, + severity: str = "error", + description: str | None = None, +): + """Decorate a DataFrame predicate as a reusable custom data check.""" + from framevitals.checks import check as _check + + return _check( + name, + severity=severity, + description=description, + ) + + +def run_checks(data: Any, checks: Any) -> CheckResult: + """Run custom data checks exactly against a dataset.""" + from framevitals.checks import run_checks as _run_checks + + return _run_checks(data, checks) + + +def discover_checks(*, group: str = "framevitals.checks"): + """Explicitly load installed third-party check plugins.""" + from framevitals.plugins import discover_checks as _discover_checks + + return _discover_checks(group=group) + + +def gate( + current: Any, + *, + reference: Any = None, + contract: Any = None, + custom_checks: Any = None, + columns: list[str] | None = None, + max_columns: int = 30, + drift_warn_on: str = "moderate", + drift_fail_on: str = "severe", + fail_on_validation_warning: bool = False, +) -> GateResult: + """Run contract, custom, and drift checks as one CI-friendly quality verdict.""" + from framevitals.operations import gate as _gate + + return _gate( + current, + reference=reference, + contract=contract, + custom_checks=custom_checks, + columns=columns, + max_columns=max_columns, + drift_warn_on=drift_warn_on, + drift_fail_on=drift_fail_on, + fail_on_validation_warning=fail_on_validation_warning, + ) + + +__all__ = [ + "AnalysisConfig", "AnalysisPlan", "AnalysisResult", "AnalysisSnapshot", "SnapshotHistory", + "CleaningPlan", "ColumnResult", "DiagnosticResult", "DataCheck", "CheckResult", "DriftResult", + "GateResult", "ValidationResult", "inspect_source", "profile", "roles", "health", + "ml_readiness", "quality", "statistics", "anomalies", "relationships", "system_info", + "target_analysis", "analyze", "plan", "plan_cleaning", "clean", "compare", "infer_contract", + "validate", "check", "run_checks", "discover_checks", "gate", "available_modules", + "create_snapshot", "load_snapshot", "compare_snapshots", "__version__", +] diff --git a/src/framevitals/acceleration.py b/src/framevitals/acceleration.py new file mode 100644 index 0000000..805c4e8 --- /dev/null +++ b/src/framevitals/acceleration.py @@ -0,0 +1,238 @@ +"""Hardware and optional acceleration discovery for FrameVitals. + +Discovery is intentionally read-only: importing or calling this module never +installs packages and never makes CUDA a hard dependency. The future planner can +consume this structured capability report when selecting per-operation backends. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from importlib.util import find_spec +import os +import platform +import re +import shutil +import subprocess +from typing import Any + + +_CUDA_VERSION_RE = re.compile(r"CUDA Version:\s*([0-9]+)(?:\.([0-9]+))?") + + +@dataclass(frozen=True, slots=True) +class GpuDevice: + name: str + memory_total_mb: int | None = None + driver_version: str | None = None + + +@dataclass(frozen=True, slots=True) +class SystemCapabilities: + platform: str + architecture: str + cpu_count: int | None + memory_total_bytes: int | None + native_module_available: bool + nvidia_driver_available: bool + cuda_compatibility: str | None + gpu_devices: tuple[GpuDevice, ...] = field(default_factory=tuple) + cupy_installed: bool = False + cupy_usable: bool = False + cupy_error: str | None = None + recommended_cupy_package: str | None = None + default_cpu_backend: str = "numpy" + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["gpu_devices"] = [asdict(device) for device in self.gpu_devices] + payload["eligible_backends"] = [self.default_cpu_backend] + if self.cupy_usable: + payload["eligible_backends"].append("cupy") + return payload + + +def _total_memory_bytes() -> int | None: + """Best-effort physical-memory detection using only the standard library.""" + try: + if os.name == "nt": + import ctypes + + class MemoryStatusEx(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("ullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + status = MemoryStatusEx() + status.dwLength = ctypes.sizeof(MemoryStatusEx) + if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + return int(status.ullTotalPhys) + return None + + page_size = os.sysconf("SC_PAGE_SIZE") + physical_pages = os.sysconf("SC_PHYS_PAGES") + return int(page_size * physical_pages) + except (AttributeError, OSError, ValueError): + return None + + +def _run_nvidia_smi(arguments: list[str]) -> str | None: + executable = shutil.which("nvidia-smi") + if executable is None: + return None + try: + completed = subprocess.run( + [executable, *arguments], + capture_output=True, + text=True, + timeout=3, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if completed.returncode != 0: + return None + return completed.stdout.strip() + + +def _detect_nvidia() -> tuple[tuple[GpuDevice, ...], str | None]: + query = _run_nvidia_smi([ + "--query-gpu=name,memory.total,driver_version", + "--format=csv,noheader,nounits", + ]) + if not query: + return (), None + + devices: list[GpuDevice] = [] + for line in query.splitlines(): + parts = [part.strip() for part in line.split(",")] + if not parts or not parts[0]: + continue + memory = None + if len(parts) >= 2: + try: + memory = int(float(parts[1])) + except ValueError: + memory = None + devices.append( + GpuDevice( + name=parts[0], + memory_total_mb=memory, + driver_version=parts[2] if len(parts) >= 3 and parts[2] else None, + ) + ) + + banner = _run_nvidia_smi([]) + cuda_compatibility = None + if banner: + match = _CUDA_VERSION_RE.search(banner) + if match: + major = match.group(1) + minor = match.group(2) or "0" + cuda_compatibility = f"{major}.{minor}" + + return tuple(devices), cuda_compatibility + + +def _recommend_cupy_package( + system: str, + cuda_compatibility: str | None, + *, + has_nvidia: bool, +) -> str | None: + """Return the current CuPy wheel family when the environment is eligible.""" + if not has_nvidia or system not in {"Linux", "Windows"}: + return None + if not cuda_compatibility: + return None + + try: + major = int(cuda_compatibility.split(".", 1)[0]) + except ValueError: + return None + + if major >= 13: + return "cupy-cuda13x[ctk]" + if major == 12: + return "cupy-cuda12x[ctk]" + return None + + +def _probe_cupy() -> tuple[bool, bool, str | None]: + installed = find_spec("cupy") is not None + if not installed: + return False, False, None + + try: + import cupy + + device_count = int(cupy.cuda.runtime.getDeviceCount()) + return True, device_count > 0, None if device_count > 0 else "No CUDA devices found." + except Exception as exc: # noqa: BLE001 - capability probe must fail soft + return True, False, f"{type(exc).__name__}: {exc}" + + +def detect_system_capabilities(*, probe_gpu: bool = True) -> SystemCapabilities: + """Inspect CPU/native/CUDA capabilities without installing anything.""" + system = platform.system() or "Unknown" + architecture = platform.machine() or "unknown" + native_available = find_spec("framevitals._native") is not None + + devices: tuple[GpuDevice, ...] = () + cuda_compatibility = None + if probe_gpu and system != "Darwin": + devices, cuda_compatibility = _detect_nvidia() + + if probe_gpu: + cupy_installed, cupy_usable, cupy_error = _probe_cupy() + else: + cupy_installed = find_spec("cupy") is not None + cupy_usable = False + cupy_error = None + + recommendation = _recommend_cupy_package( + system, + cuda_compatibility, + has_nvidia=bool(devices), + ) + + return SystemCapabilities( + platform=system, + architecture=architecture, + cpu_count=os.cpu_count(), + memory_total_bytes=_total_memory_bytes(), + native_module_available=native_available, + nvidia_driver_available=bool(devices), + cuda_compatibility=cuda_compatibility, + gpu_devices=devices, + cupy_installed=cupy_installed, + cupy_usable=cupy_usable, + cupy_error=cupy_error, + recommended_cupy_package=recommendation, + default_cpu_backend="rust" if native_available else "numpy", + ) + + +def system_info(*, probe_gpu: bool = True) -> dict[str, Any]: + """Return a JSON-safe report suitable for API/CLI/TUI diagnostics.""" + capabilities = detect_system_capabilities(probe_gpu=probe_gpu) + payload = capabilities.to_dict() + payload["gpu_acceleration"] = { + "available": capabilities.cupy_usable, + "installable": bool( + capabilities.nvidia_driver_available + and capabilities.recommended_cupy_package + and not capabilities.cupy_installed + ), + "recommended_package": capabilities.recommended_cupy_package, + "automatic_install_performed": False, + } + return payload diff --git a/src/framevitals/advanced_indicators.py b/src/framevitals/advanced_indicators.py index f6b7b93..7291a7c 100644 --- a/src/framevitals/advanced_indicators.py +++ b/src/framevitals/advanced_indicators.py @@ -1,10 +1,35 @@ +from __future__ import annotations + +import re + import numpy as np import pandas as pd -SENSITIVE_KEYWORDS = [ - "gender", "sex", "age", "race", "religion", - "region", "location", "income", "salary", "caste", -] +SENSITIVE_KEYWORDS = { + "gender", + "sex", + "age", + "race", + "religion", + "region", + "location", + "income", + "salary", + "caste", +} + + +def _column_tokens(name: object) -> set[str]: + normalized = re.sub(r"[^a-z0-9]+", "_", str(name).strip().lower()).strip("_") + return {token for token in normalized.split("_") if token} + + +def _bounded_non_null_sample(series: pd.Series, max_rows: int = 200) -> pd.Series: + clean = series.dropna() + if len(clean) <= max_rows: + return clean + positions = np.linspace(0, len(clean) - 1, num=max_rows, dtype=np.int64) + return clean.iloc[np.unique(positions)] def calculate_column_utility(df): @@ -45,37 +70,65 @@ def calculate_column_utility(df): def calculate_anomalies(df): + """Calculate simple IQR anomaly density using O(rows) auxiliary memory.""" numeric = df.select_dtypes(include=[np.number]) if numeric.empty: return {"anomalous_rows": 0, "highest_score": 0, "top_rows": []} - flags = pd.DataFrame(index=df.index) + # The old implementation materialized one boolean column per numeric + # feature. A single counter vector produces the identical row score while + # avoiding O(rows * numeric_columns) temporary memory. + row_hits = np.zeros(len(df), dtype=np.uint32) + usable_columns = 0 for col in numeric.columns: - q1 = numeric[col].quantile(0.25) - q3 = numeric[col].quantile(0.75) + series = numeric[col] + q1 = series.quantile(0.25) + q3 = series.quantile(0.75) iqr = q3 - q1 if pd.isna(iqr) or iqr == 0: - flags[col] = False continue lower = q1 - 1.5 * iqr upper = q3 + 1.5 * iqr - flags[col] = (numeric[col] < lower) | (numeric[col] > upper) + mask = ((series < lower) | (series > upper)).to_numpy(dtype=bool) + row_hits += mask + usable_columns += 1 - scores = flags.sum(axis=1) / max(len(numeric.columns), 1) - top = scores.sort_values(ascending=False).head(10) + denominator = max(usable_columns, 1) + scores = row_hits.astype(np.float64) / denominator + anomalous_mask = row_hits > 0 + + if not anomalous_mask.any(): + return {"anomalous_rows": 0, "highest_score": 0, "top_rows": []} + + top_count = min(10, len(scores)) + candidate_positions = np.argpartition(scores, -top_count)[-top_count:] + candidate_positions = candidate_positions[ + np.argsort(scores[candidate_positions])[::-1] + ] + + top_rows = [] + for position in candidate_positions: + score = float(scores[position]) + if score <= 0: + continue + index_value = df.index[int(position)] + top_rows.append({ + "row_index": ( + int(index_value) + if isinstance(index_value, (int, np.integer)) + else str(index_value) + ), + "score": round(score, 3), + }) return { - "anomalous_rows": int((scores > 0).sum()), + "anomalous_rows": int(anomalous_mask.sum()), "highest_score": round(float(scores.max()), 3), - "top_rows": [ - {"row_index": int(idx), "score": round(float(score), 3)} - for idx, score in top.items() - if score > 0 - ], + "top_rows": top_rows, } @@ -83,8 +136,8 @@ def detect_fairness_review(df): found = [] for col in df.columns: - lower_col = col.lower() - if any(keyword in lower_col for keyword in SENSITIVE_KEYWORDS): + tokens = _column_tokens(col) + if tokens & SENSITIVE_KEYWORDS: found.append(col) if found: @@ -102,21 +155,34 @@ def detect_fairness_review(df): def calculate_freshness(df): - date_columns = [] + """Detect a date column with bounded screening and one full parse at most.""" + candidates: list[tuple[float, str]] = [] for col in df.columns: - if pd.api.types.is_numeric_dtype(df[col]): + series = df[col] + if pd.api.types.is_numeric_dtype(series): continue - parsed = pd.to_datetime(df[col], errors="coerce", format="mixed") + sample = _bounded_non_null_sample(series, 200) + if sample.empty: + continue + + parsed_sample = pd.to_datetime(sample, errors="coerce", format="mixed") + parse_rate = float(parsed_sample.notna().mean()) + if parse_rate >= 0.7: + candidates.append((parse_rate, col)) - if parsed.notna().mean() >= 0.7: - date_columns.append((col, parsed)) + if not candidates: + return {"available": False, "message": "No strong date column detected."} - if not date_columns: + # Parse only the strongest candidate across the complete column. Future + # Arrow/Rust sources will calculate extrema while streaming instead. + candidates.sort(key=lambda item: (-item[0], item[1])) + _, col = candidates[0] + parsed = pd.to_datetime(df[col], errors="coerce", format="mixed") + if parsed.notna().mean() < 0.7: return {"available": False, "message": "No strong date column detected."} - col, parsed = date_columns[0] min_date = parsed.min() max_date = parsed.max() diff --git a/src/framevitals/ai_agent.py b/src/framevitals/ai_agent.py index 59383e2..cc9fab9 100644 --- a/src/framevitals/ai_agent.py +++ b/src/framevitals/ai_agent.py @@ -1,5 +1,5 @@ """ -DataLens AI — Agentic AI Layer (WS-9) +FrameVitals — Agentic AI Layer (WS-9) ====================================== Planner → Executor → Critic → Writer loop running on a local Ollama model with OpenRouter and a deterministic fallback. @@ -27,7 +27,6 @@ from typing import Any import pandas as pd -import pydantic from pydantic import BaseModel, Field, ValidationError from framevitals.rag_index import ( @@ -54,7 +53,7 @@ ) _OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" _OPENROUTER_SITE_URL = os.environ.get("OPENROUTER_SITE_URL", "http://127.0.0.1:5055") -_OPENROUTER_APP_NAME = os.environ.get("OPENROUTER_APP_NAME", "DataLens AI") +_OPENROUTER_APP_NAME = os.environ.get("OPENROUTER_APP_NAME", "FrameVitals") # Per-call Ollama timeout. Cloud models (deepseek 671b, gpt-oss 120b, kimi) # can take 30-60s end-to-end. Override with DATALENS_OLLAMA_TIMEOUT (seconds). diff --git a/src/framevitals/analysis_api.py b/src/framevitals/analysis_api.py new file mode 100644 index 0000000..9e0b5e7 --- /dev/null +++ b/src/framevitals/analysis_api.py @@ -0,0 +1,173 @@ +"""Public full-analysis dispatcher. + +This module keeps input/source routing separate from the legacy analysis API. +Streaming-capable sources use the bounded streaming orchestrator when artifacts +are disabled; DataFrames and exact/materialized execution retain the existing +full pipeline behavior. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pandas as pd + +from framevitals.config import ConfigInput, resolve_config +from framevitals.pipeline import run_full_analysis +from framevitals.result import AnalysisResult +from framevitals.sources import StreamingDatasetSource, resolve_source + + +DataInput = Any + + +_MODE_DISABLED_MODULES: dict[str, frozenset[str]] = { + # Quick is intentionally an overview. Keep explicit target intelligence and + # artifact cleaning available for backwards compatibility, while omitting + # the heavier anomaly/time-series/research/modeling layers. + "quick": frozenset({ + "deep_statistics", + "anomaly_detection", + "time_series", + "text_profile", + "modeling", + "explainability", + }), + # Standard is the operational default. It keeps practical anomaly, + # time-series and target diagnostics, but leaves research-grade statistics, + # free-text profiling and model training/explainability to deeper tiers. + "standard": frozenset({ + "deep_statistics", + "text_profile", + "modeling", + "explainability", + }), + # Deep is the advanced diagnostic tier: research-grade statistics and text + # profiling are enabled, while repeated model CV/refitting is reserved for + # research mode where the extra runtime is an explicit user choice. + "deep": frozenset({"modeling", "explainability"}), + "research": frozenset(), +} + + +def _effective_disabled_modules( + mode: str, + user_disabled: tuple[str, ...], +) -> tuple[str, ...]: + """Merge explicit disables with the stable module policy for a mode.""" + implicit = _MODE_DISABLED_MODULES.get(mode) + if implicit is None: + raise ValueError(f"Unknown analysis mode: {mode}") + return tuple(sorted(set(user_disabled) | set(implicit))) + + +def analyze( + data: DataInput, + *, + target: str | None = None, + mode: str | None = None, + artifacts: bool | None = None, + workers: int | None = None, + preset: str | None = None, + config: ConfigInput = None, + disabled_modules: list[str] | tuple[str, ...] | None = None, +) -> AnalysisResult: + """Analyze a tabular dataset through the appropriate execution source.""" + resolved = resolve_config( + config, + preset=preset, + mode=mode, + target=target, + artifacts=artifacts, + workers=workers, + disabled_modules=disabled_modules, + ) + effective_disabled = _effective_disabled_modules( + resolved.mode, + resolved.disabled_modules, + ) + dataset_id = f"fv_{uuid4().hex[:12]}" + + # Preserve the direct DataFrame path so callers do not pay for a defensive + # source-layer copy before the established materialized pipeline begins. + if isinstance(data, pd.DataFrame): + if data.empty: + raise ValueError("Dataset DataFrame is empty.") + payload = run_full_analysis( + dataset_id=dataset_id, + original_filename="", + analysis_mode=resolved.mode, + target_column=resolved.target, + parallel_workers=resolved.workers, + skip_ai=True, + dataframe=data, + write_artifacts=resolved.artifacts, + disabled_modules=effective_disabled, + ) + else: + source = resolve_source(data) + metadata = source.inspect() + if metadata.rows == 0: + raise ValueError(f"Dataset is empty: {metadata.name}") + + if ( + metadata.supports_streaming + and isinstance(source, StreamingDatasetSource) + and not resolved.artifacts + ): + from framevitals.streaming_exact_reuse import reuse_streaming_exact_statistics + from framevitals.streaming_pipeline import run_streaming_analysis + + payload = run_streaming_analysis( + source=source, + dataset_id=dataset_id, + original_filename=metadata.name, + analysis_mode=resolved.mode, + target_column=resolved.target, + parallel_workers=resolved.workers, + skip_ai=True, + disabled_modules=effective_disabled, + ) + payload = reuse_streaming_exact_statistics(payload) + elif isinstance(data, (str, Path)): + path = Path(data) + if not path.exists(): + raise FileNotFoundError(f"Dataset not found: {path}") + if not path.is_file(): + raise ValueError(f"Expected a file for dataset, got: {path}") + payload = run_full_analysis( + dataset_id=dataset_id, + file_path=path, + original_filename=metadata.name, + analysis_mode=resolved.mode, + target_column=resolved.target, + parallel_workers=resolved.workers, + skip_ai=True, + write_artifacts=resolved.artifacts, + disabled_modules=effective_disabled, + ) + else: + dataframe = source.load() + payload = run_full_analysis( + dataset_id=dataset_id, + original_filename=metadata.name, + analysis_mode=resolved.mode, + target_column=resolved.target, + parallel_workers=resolved.workers, + skip_ai=True, + dataframe=dataframe, + write_artifacts=resolved.artifacts, + disabled_modules=effective_disabled, + ) + + # Keep public configuration/provenance compatible: ``disabled_modules`` + # describes only explicit caller configuration. Mode policy is observable + # through ``execution.module_status`` and analysis selection instead of + # masquerading as a user-supplied disable list. + payload["config"] = resolved.to_dict() + execution = payload.get("execution") + if isinstance(execution, dict): + execution["disabled_modules"] = sorted(resolved.disabled_modules) + return AnalysisResult(payload) diff --git a/src/framevitals/analysis_inventory.py b/src/framevitals/analysis_inventory.py index 3a8a250..aee09c6 100644 --- a/src/framevitals/analysis_inventory.py +++ b/src/framevitals/analysis_inventory.py @@ -88,15 +88,15 @@ "requires": {}, "outputs": ["warnings", "severity"]}, {"id": "target_analysis", "name": "Target Column Analysis", "category": "Machine Learning", - "modes": ["deep", "research"], "priority": "high", + "modes": ["standard", "deep", "research"], "priority": "high", "requires": {}, "requires_user_target": True, "outputs": ["target_candidates", "task_type"]}, {"id": "feature_importance", "name": "Feature Importance Analysis", "category": "Machine Learning", - "modes": ["deep", "research"], "priority": "high", + "modes": ["research"], "priority": "high", "requires": {"has_numeric_columns": True}, "requires_user_target": True, "outputs": ["ranking", "chart"]}, {"id": "baseline_model", "name": "Baseline Model Analysis", "category": "Machine Learning", - "modes": ["deep", "research"], "priority": "high", + "modes": ["research"], "priority": "high", "requires": {"has_numeric_columns": True}, "requires_user_target": True, "outputs": ["metrics"]}, {"id": "anomaly_detection", "name": "Anomaly Detection", "category": "Anomaly", @@ -113,7 +113,7 @@ "outputs": ["privacy_score", "pii_warning"]}, {"id": "time_series_signal", "name": "Time Series Structure Signal", "category": "Temporal", - "modes": ["deep", "research"], "priority": "medium", + "modes": ["standard", "deep", "research"], "priority": "medium", "requires": {"has_time_series_structure": True}, "outputs": ["warning", "recommendation"]}, {"id": "ai_summary", "name": "AI Insight Generation", "category": "AI", diff --git a/src/framevitals/analysis_state.py b/src/framevitals/analysis_state.py new file mode 100644 index 0000000..8ce5f0b --- /dev/null +++ b/src/framevitals/analysis_state.py @@ -0,0 +1,245 @@ +"""Mergeable analysis-state primitives. + +These classes are the reference semantics for FrameVitals' Rust/Arrow streaming +engine. A partition can be summarized independently and merged with another +partition without concatenating raw rows. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from math import sqrt +from typing import Any + +import numpy as np +import pandas as pd + +from framevitals.backends import numeric_state + + +@dataclass(slots=True) +class NumericColumnState: + """Mergeable exact central-moment state through fourth order.""" + + count: int = 0 + missing: int = 0 + mean: float = 0.0 + m2: float = 0.0 + m3: float = 0.0 + m4: float = 0.0 + minimum: float | None = None + maximum: float | None = None + infinite: int = 0 + + @classmethod + def from_series(cls, series: pd.Series) -> "NumericColumnState": + payload = numeric_state(series) + count = int(payload["count"]) + variance = payload.get("variance") + return cls( + count=count, + missing=int(payload["missing"]), + mean=float(payload["mean"]) if count else 0.0, + m2=float( + payload.get( + "m2", + float(variance) * (count - 1) + if variance is not None and count >= 2 + else 0.0, + ) + ), + m3=float(payload.get("m3", 0.0)), + m4=float(payload.get("m4", 0.0)), + minimum=( + float(payload["minimum"]) + if payload.get("minimum") is not None + else None + ), + maximum=( + float(payload["maximum"]) + if payload.get("maximum") is not None + else None + ), + infinite=int(payload["infinite"]), + ) + + def merge(self, other: "NumericColumnState") -> "NumericColumnState": + """Merge another partition through fourth central moment.""" + if other.count == 0: + return NumericColumnState( + count=self.count, + missing=self.missing + other.missing, + mean=self.mean, + m2=self.m2, + m3=self.m3, + m4=self.m4, + minimum=self.minimum, + maximum=self.maximum, + infinite=self.infinite + other.infinite, + ) + if self.count == 0: + return NumericColumnState( + count=other.count, + missing=self.missing + other.missing, + mean=other.mean, + m2=other.m2, + m3=other.m3, + m4=other.m4, + minimum=other.minimum, + maximum=other.maximum, + infinite=self.infinite + other.infinite, + ) + + total = self.count + other.count + left = float(self.count) + right = float(other.count) + total_f = float(total) + delta = other.mean - self.mean + delta2 = delta * delta + delta3 = delta2 * delta + delta4 = delta2 * delta2 + total2 = total_f * total_f + total3 = total2 * total_f + + mean = self.mean + delta * right / total_f + m2 = self.m2 + other.m2 + delta2 * left * right / total_f + m3 = ( + self.m3 + + other.m3 + + delta3 * left * right * (left - right) / total2 + + 3.0 * delta * (left * other.m2 - right * self.m2) / total_f + ) + m4 = ( + self.m4 + + other.m4 + + delta4 + * left + * right + * (left * left - left * right + right * right) + / total3 + + 6.0 + * delta2 + * (left * left * other.m2 + right * right * self.m2) + / total2 + + 4.0 * delta * (left * other.m3 - right * self.m3) / total_f + ) + minimum = min( + value for value in (self.minimum, other.minimum) if value is not None + ) + maximum = max( + value for value in (self.maximum, other.maximum) if value is not None + ) + return NumericColumnState( + count=total, + missing=self.missing + other.missing, + mean=float(mean), + m2=float(m2), + m3=float(m3), + m4=float(m4), + minimum=float(minimum), + maximum=float(maximum), + infinite=self.infinite + other.infinite, + ) + + @property + def variance(self) -> float | None: + if self.count < 2: + return None + return self.m2 / (self.count - 1) + + @property + def std(self) -> float | None: + variance = self.variance + return sqrt(variance) if variance is not None and variance >= 0 else None + + @property + def skewness(self) -> float | None: + """Bias-corrected Fisher-Pearson sample skewness.""" + if self.count < 3 or self.m2 <= 0: + return None + n = float(self.count) + population_skew = sqrt(n) * self.m3 / (self.m2 ** 1.5) + return sqrt(n * (n - 1.0)) / (n - 2.0) * population_skew + + @property + def kurtosis(self) -> float | None: + """Bias-corrected Fisher excess kurtosis.""" + if self.count < 4 or self.m2 <= 0: + return None + n = float(self.count) + population_excess = n * self.m4 / (self.m2 * self.m2) - 3.0 + return ( + (n - 1.0) + / ((n - 2.0) * (n - 3.0)) + * ((n + 1.0) * population_excess + 6.0) + ) + + def to_dict(self) -> dict[str, Any]: + return { + "count": self.count, + "missing": self.missing, + "infinite": self.infinite, + "mean": self.mean if self.count else None, + "variance": self.variance, + "std": self.std, + "skewness": self.skewness, + "kurtosis": self.kurtosis, + "m2": self.m2, + "m3": self.m3, + "m4": self.m4, + "minimum": self.minimum, + "maximum": self.maximum, + } + + +@dataclass(slots=True) +class AnalysisState: + """Compact mergeable state for a dataset partition.""" + + rows: int = 0 + columns: int = 0 + numeric: dict[str, NumericColumnState] = field(default_factory=dict) + schema: dict[str, str] = field(default_factory=dict) + + @classmethod + def from_frame(cls, dataframe: pd.DataFrame) -> "AnalysisState": + numeric_columns = dataframe.select_dtypes(include=[np.number]).columns + return cls( + rows=int(len(dataframe)), + columns=int(len(dataframe.columns)), + numeric={ + str(column): NumericColumnState.from_series(dataframe[column]) + for column in numeric_columns + }, + schema={str(column): str(dtype) for column, dtype in dataframe.dtypes.items()}, + ) + + def merge(self, other: "AnalysisState") -> "AnalysisState": + """Merge states without access to either partition's raw rows.""" + if self.schema and other.schema and self.schema != other.schema: + raise ValueError("Cannot merge AnalysisState objects with different schemas.") + + names = set(self.numeric) | set(other.numeric) + merged_numeric: dict[str, NumericColumnState] = {} + for name in names: + left = self.numeric.get(name, NumericColumnState()) + right = other.numeric.get(name, NumericColumnState()) + merged_numeric[name] = left.merge(right) + + return AnalysisState( + rows=self.rows + other.rows, + columns=max(self.columns, other.columns), + numeric=merged_numeric, + schema=dict(self.schema or other.schema), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "rows": self.rows, + "columns": self.columns, + "schema": dict(self.schema), + "numeric": { + name: state.to_dict() + for name, state in sorted(self.numeric.items()) + }, + } diff --git a/src/framevitals/anomaly_ensemble.py b/src/framevitals/anomaly_ensemble.py index 30dfebe..87af125 100644 --- a/src/framevitals/anomaly_ensemble.py +++ b/src/framevitals/anomaly_ensemble.py @@ -1,28 +1,17 @@ -""" -Anomaly Ensemble (WS-2) -======================= -Multi-detector anomaly scoring for tabular data. - -Detectors: - 1. IsolationForest (sklearn.ensemble) - 2. LocalOutlierFactor (sklearn.neighbors) - 3. EllipticEnvelope (sklearn.covariance) — robust Gaussian - 4. Robust z-score (MAD) (numpy) - 5. Mahalanobis distance (sklearn shrinkage covariance) - 6. ECOD (pyod, optional) - 7. COPOD (pyod, optional) - -Each detector emits a normalized score in [0, 1] (higher = more anomalous). -The ensemble score is the column-wise mean across available detectors. - -Public entry point: - detect_anomalies_ensemble(df, contamination=0.05) -> dict +"""Multi-detector anomaly scoring for tabular data. + +The ensemble combines deterministic/classical detectors that are useful across +many tabular datasets and optionally adds ECOD/COPOD when PyOD is installed. +Scores remain normalized to [0, 1] and the historical mean-ensemble threshold +is preserved, while the result now also reports detector agreement, failures, +input preparation, and feature-level context for top anomalous rows. """ from __future__ import annotations +import math import warnings -from typing import Any +from typing import Any, Callable import numpy as np import pandas as pd @@ -34,10 +23,6 @@ from sklearn.preprocessing import StandardScaler -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - def _to_unit(scores: np.ndarray) -> np.ndarray: """Min-max normalize an arbitrary score vector to [0, 1].""" s = np.asarray(scores, dtype=float) @@ -49,44 +34,84 @@ def _to_unit(scores: np.ndarray) -> np.ndarray: if hi - lo < 1e-12: return np.zeros_like(s) out = (s - lo) / (hi - lo) - out = np.where(np.isnan(out), 0.0, out) - return out + return np.where(np.isnan(out), 0.0, out) -def _prepare_numeric_matrix(df: pd.DataFrame, max_columns: int = 30): - """ - Select numeric columns, drop constants, impute medians, scale. - Returns (X_scaled_df, used_columns) or (None, []) if not enough columns. - """ +def _prepare_numeric_matrix( + df: pd.DataFrame, + max_columns: int = 30, +) -> tuple[pd.DataFrame | None, list[str], dict[str, Any]]: + """Select, sanitize, impute, and scale numeric columns.""" numeric = df.select_dtypes(include=[np.number]).copy() - - # Drop constants - for col in list(numeric.columns): - if numeric[col].nunique(dropna=True) <= 1: - numeric = numeric.drop(columns=[col]) + original_numeric_columns = list(numeric.columns) + + infinity_counts: dict[str, int] = {} + for column in list(numeric.columns): + values = pd.to_numeric(numeric[column], errors="coerce") + finite_array = values.to_numpy(dtype="float64", na_value=np.nan) + inf_count = int(np.isinf(finite_array).sum()) + if inf_count: + infinity_counts[column] = inf_count + numeric[column] = values.replace([np.inf, -np.inf], np.nan) + + dropped_all_missing: list[str] = [] + dropped_constant: list[str] = [] + for column in list(numeric.columns): + if numeric[column].notna().sum() == 0: + numeric = numeric.drop(columns=[column]) + dropped_all_missing.append(column) + elif numeric[column].nunique(dropna=True) <= 1: + numeric = numeric.drop(columns=[column]) + dropped_constant.append(column) if numeric.shape[1] < 1: - return None, [] + metadata = { + "numeric_columns_found": len(original_numeric_columns), + "used_columns": [], + "dropped_constant_columns": dropped_constant, + "dropped_all_missing_columns": dropped_all_missing, + "infinite_values_replaced": infinity_counts, + "missing_values_imputed": {}, + "truncated_columns": False, + } + return None, [], metadata - # Cap dimensionality (keep most-populated columns) - if numeric.shape[1] > max_columns: + truncated = numeric.shape[1] > max_columns + if truncated: keep = ( - numeric.notna().sum().sort_values(ascending=False).head(max_columns).index.tolist() + numeric.notna() + .sum() + .sort_values(ascending=False) + .head(max_columns) + .index.tolist() ) numeric = numeric[keep] - # Median impute then scale + missing_counts = { + column: int(count) + for column, count in numeric.isna().sum().items() + if int(count) > 0 + } + imputer = SimpleImputer(strategy="median") scaler = StandardScaler() X = imputer.fit_transform(numeric) X = scaler.fit_transform(X) - return pd.DataFrame(X, columns=numeric.columns, index=df.index), list(numeric.columns) - + used_columns = list(numeric.columns) + metadata = { + "numeric_columns_found": len(original_numeric_columns), + "used_columns": used_columns, + "dropped_constant_columns": dropped_constant, + "dropped_all_missing_columns": dropped_all_missing, + "infinite_values_replaced": { + key: value for key, value in infinity_counts.items() if key in used_columns + }, + "missing_values_imputed": missing_counts, + "truncated_columns": truncated, + } + return pd.DataFrame(X, columns=used_columns, index=df.index), used_columns, metadata -# --------------------------------------------------------------------------- -# Individual detectors -# --------------------------------------------------------------------------- def _detect_iforest(X: np.ndarray, contamination: float) -> np.ndarray: model = IsolationForest( @@ -96,26 +121,25 @@ def _detect_iforest(X: np.ndarray, contamination: float) -> np.ndarray: n_jobs=-1, ) model.fit(X) - # score_samples: higher = more normal. Negate so higher = more anomalous. - raw = -model.score_samples(X) - return _to_unit(raw) + return _to_unit(-model.score_samples(X)) def _detect_lof(X: np.ndarray) -> np.ndarray: - model = LocalOutlierFactor(n_neighbors=min(20, max(5, X.shape[0] // 20)), n_jobs=-1) + n_neighbors = min(20, max(5, X.shape[0] // 20)) + n_neighbors = min(n_neighbors, max(X.shape[0] - 1, 1)) + model = LocalOutlierFactor(n_neighbors=n_neighbors, n_jobs=-1) model.fit_predict(X) - raw = -model.negative_outlier_factor_ # higher = more anomalous - return _to_unit(raw) + return _to_unit(-model.negative_outlier_factor_) def _detect_elliptic(X: np.ndarray, contamination: float) -> np.ndarray: - try: - model = EllipticEnvelope(contamination=contamination, support_fraction=None, random_state=42) - model.fit(X) - raw = -model.score_samples(X) - return _to_unit(raw) - except Exception: - return np.zeros(X.shape[0]) + model = EllipticEnvelope( + contamination=contamination, + support_fraction=None, + random_state=42, + ) + model.fit(X) + return _to_unit(-model.score_samples(X)) def _detect_mad_robust_z(X: np.ndarray) -> np.ndarray: @@ -124,50 +148,70 @@ def _detect_mad_robust_z(X: np.ndarray) -> np.ndarray: mad = np.median(np.abs(X - median), axis=0) mad_safe = np.where(mad > 0, mad, 1.0) z = np.abs((X - median) / (1.4826 * mad_safe)) - raw = z.mean(axis=1) - return _to_unit(raw) + return _to_unit(z.mean(axis=1)) def _detect_mahalanobis(X: np.ndarray) -> np.ndarray: - """Mahalanobis distance using a shrinkage / robust covariance estimate.""" + """Mahalanobis distance using robust covariance with a stable fallback.""" try: cov = MinCovDet(random_state=42).fit(X) raw = cov.mahalanobis(X) return _to_unit(raw) except Exception: - # Fallback to plain covariance - try: - mean = X.mean(axis=0) - cov_matrix = np.cov(X.T) + np.eye(X.shape[1]) * 1e-6 - inv = np.linalg.pinv(cov_matrix) - diff = X - mean - raw = np.einsum("ij,jk,ik->i", diff, inv, diff) - return _to_unit(np.sqrt(np.maximum(raw, 0))) - except Exception: - return np.zeros(X.shape[0]) - - -def _detect_pyod(X: np.ndarray, model_name: str) -> np.ndarray | None: - """Try ECOD or COPOD from pyod; return None if pyod is not installed.""" - try: - if model_name == "ecod": - from pyod.models.ecod import ECOD as Model - elif model_name == "copod": - from pyod.models.copod import COPOD as Model - else: - return None - model = Model() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - model.fit(X) - return _to_unit(model.decision_scores_) - except Exception: - return None + mean = X.mean(axis=0) + cov_matrix = np.atleast_2d(np.cov(X.T)) + cov_matrix = cov_matrix + np.eye(X.shape[1]) * 1e-6 + inv = np.linalg.pinv(cov_matrix) + diff = X - mean + raw = np.einsum("ij,jk,ik->i", diff, inv, diff) + return _to_unit(np.sqrt(np.maximum(raw, 0))) + + +def _detect_pyod(X: np.ndarray, model_name: str) -> np.ndarray: + if model_name == "ecod": + from pyod.models.ecod import ECOD as Model + elif model_name == "copod": + from pyod.models.copod import COPOD as Model + else: + raise ValueError(f"Unsupported PyOD detector: {model_name}") + + model = Model() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + model.fit(X) + return _to_unit(model.decision_scores_) -# --------------------------------------------------------------------------- -# Public entry point -# --------------------------------------------------------------------------- +def _detector_summary(scores: np.ndarray) -> dict[str, float]: + values = np.asarray(scores, dtype=float) + return { + "mean": round(float(np.mean(values)), 4), + "median": round(float(np.median(values)), 4), + "p95": round(float(np.quantile(values, 0.95)), 4), + "p99": round(float(np.quantile(values, 0.99)), 4), + "max": round(float(np.max(values)), 4), + } + + +def _top_feature_deviations( + X_df: pd.DataFrame, + row_index: Any, + *, + limit: int = 3, +) -> list[dict[str, Any]]: + row = X_df.loc[row_index] + if isinstance(row, pd.DataFrame): + row = row.iloc[0] + ordered = row.abs().sort_values(ascending=False).head(limit) + return [ + { + "feature": str(feature), + "standardized_deviation": round(float(abs(row[feature])), 4), + "direction": "high" if float(row[feature]) >= 0 else "low", + } + for feature in ordered.index + ] + def detect_anomalies_ensemble( df: pd.DataFrame, @@ -175,35 +219,25 @@ def detect_anomalies_ensemble( threshold: float = 0.6, max_columns: int = 30, top_k: int = 25, -) -> dict: - """ - Run all available detectors and return a JSON-safe summary. - - Args: - df: Input dataframe. - contamination: Expected anomaly fraction passed to detectors that need it. - threshold: Ensemble score above which a row is flagged. - max_columns: Cap on numeric dimensionality. - top_k: Number of top rows to include in the output payload. - - Returns shape: - { - "available": bool, - "n_rows_scored": int, - "used_columns": [...], - "detectors_run": [...], - "threshold": float, - "flagged_count": int, - "ensemble_summary": {"min","mean","median","max","p95","p99"}, - "top_rows": [{"row_index", "ensemble", "": score, ...}], - "score_table": pandas-records (top_k rows only), - } - """ - X_df, used_cols = _prepare_numeric_matrix(df, max_columns=max_columns) +) -> dict[str, Any]: + """Run available anomaly detectors and return an explainable JSON-safe summary.""" + if max_columns < 1: + raise ValueError("max_columns must be at least 1.") + if top_k < 1: + raise ValueError("top_k must be at least 1.") + if not 0 <= threshold <= 1: + raise ValueError("threshold must be between 0 and 1.") + + contamination = float(np.clip(contamination, 0.001, 0.5)) + X_df, used_cols, preparation = _prepare_numeric_matrix( + df, + max_columns=max_columns, + ) if X_df is None or X_df.empty: return { "available": False, "message": "No usable numeric columns (need at least 1 non-constant numeric column).", + "preparation": preparation, } X = X_df.values @@ -213,72 +247,89 @@ def detect_anomalies_ensemble( "available": False, "message": "Need at least 20 rows for anomaly ensemble.", "used_columns": used_cols, + "preparation": preparation, } - # Bound contamination - contamination = float(np.clip(contamination, 0.001, 0.5)) - detector_scores: dict[str, np.ndarray] = {} + detectors_failed: dict[str, str] = {} + detectors_skipped: dict[str, str] = {} - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - - try: - detector_scores["isolation_forest"] = _detect_iforest(X, contamination) - except Exception: - pass - - try: - detector_scores["local_outlier_factor"] = _detect_lof(X) - except Exception: - pass - - try: - elliptic = _detect_elliptic(X, contamination) - if elliptic.any(): - detector_scores["elliptic_envelope"] = elliptic - except Exception: - pass - + def run_detector(name: str, fn: Callable[[], np.ndarray]) -> None: try: - detector_scores["mad_robust_z"] = _detect_mad_robust_z(X) - except Exception: - pass + scores = np.asarray(fn(), dtype=float) + if scores.shape != (n_rows,): + raise ValueError( + f"Detector returned shape {scores.shape}; expected {(n_rows,)}." + ) + if not np.isfinite(scores).all(): + raise ValueError("Detector produced non-finite scores.") + detector_scores[name] = scores + except ImportError: + detectors_skipped[name] = "Optional dependency is not installed." + except Exception as exc: # noqa: BLE001 - individual detectors fail soft + detectors_failed[name] = f"{type(exc).__name__}: {exc}" - try: - mahal = _detect_mahalanobis(X) - if mahal.any(): - detector_scores["mahalanobis"] = mahal - except Exception: - pass + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + run_detector( + "isolation_forest", + lambda: _detect_iforest(X, contamination), + ) + run_detector("local_outlier_factor", lambda: _detect_lof(X)) - ecod = _detect_pyod(X, "ecod") - if ecod is not None: - detector_scores["ecod"] = ecod + if n_rows >= max(20, X.shape[1] * 2 + 1): + run_detector( + "elliptic_envelope", + lambda: _detect_elliptic(X, contamination), + ) + else: + detectors_skipped["elliptic_envelope"] = ( + "Too few rows relative to numeric dimensionality for stable covariance estimation." + ) - copod = _detect_pyod(X, "copod") - if copod is not None: - detector_scores["copod"] = copod + run_detector("mad_robust_z", lambda: _detect_mad_robust_z(X)) + run_detector("mahalanobis", lambda: _detect_mahalanobis(X)) + run_detector("ecod", lambda: _detect_pyod(X, "ecod")) + run_detector("copod", lambda: _detect_pyod(X, "copod")) if not detector_scores: return { "available": False, - "message": "All detectors failed.", + "message": "All anomaly detectors failed or were unavailable.", "used_columns": used_cols, + "preparation": preparation, + "detectors_failed": detectors_failed, + "detectors_skipped": detectors_skipped, } - # Stack detector scores into a (n_rows, n_detectors) matrix and average - score_matrix = np.column_stack(list(detector_scores.values())) + detector_names = list(detector_scores) + score_matrix = np.column_stack([detector_scores[name] for name in detector_names]) ensemble = score_matrix.mean(axis=1) - # Build per-row table (in original df index) score_df = pd.DataFrame( - {name: scores for name, scores in detector_scores.items()}, + {name: detector_scores[name] for name in detector_names}, index=df.index, ) score_df["ensemble"] = ensemble - # Summary stats + vote_thresholds: dict[str, float] = {} + vote_matrix: list[np.ndarray] = [] + for name in detector_names: + detector_threshold = float( + np.quantile(detector_scores[name], 1.0 - contamination) + ) + vote_thresholds[name] = detector_threshold + vote_matrix.append(detector_scores[name] >= detector_threshold) + + agreement_count = np.column_stack(vote_matrix).sum(axis=1) + agreement_fraction = agreement_count / len(detector_names) + score_df["agreement_count"] = agreement_count + score_df["agreement_fraction"] = agreement_fraction + + majority_required = max(1, math.ceil(len(detector_names) / 2)) + consensus_mask = agreement_count >= majority_required + flagged_mask = ensemble >= threshold + es = ensemble[np.isfinite(ensemble)] summary = { "min": float(np.min(es)), @@ -289,32 +340,58 @@ def detect_anomalies_ensemble( "p99": float(np.quantile(es, 0.99)), } - flagged_mask = score_df["ensemble"] >= threshold flagged_count = int(flagged_mask.sum()) + consensus_flagged_count = int(consensus_mask.sum()) - # Top-k rows top = score_df.sort_values("ensemble", ascending=False).head(top_k) - top_rows = [] + top_rows: list[dict[str, Any]] = [] for idx, row in top.iterrows(): - entry = {"row_index": int(idx) if isinstance(idx, (int, np.integer)) else str(idx)} - for name in detector_scores.keys(): + entry: dict[str, Any] = { + "row_index": int(idx) if isinstance(idx, (int, np.integer)) else str(idx), + } + for name in detector_names: entry[name] = round(float(row[name]), 4) entry["ensemble"] = round(float(row["ensemble"]), 4) + entry["flagged"] = bool(row["ensemble"] >= threshold) + entry["agreement_count"] = int(row["agreement_count"]) + entry["agreement_fraction"] = round(float(row["agreement_fraction"]), 4) + entry["top_feature_deviations"] = _top_feature_deviations(X_df, idx) top_rows.append(entry) + detector_summaries = { + name: _detector_summary(scores) + for name, scores in detector_scores.items() + } + return { "available": True, "n_rows_scored": int(n_rows), "used_columns": used_cols, - "detectors_run": list(detector_scores.keys()), + "preparation": preparation, + "detectors_run": detector_names, + "detectors_failed": detectors_failed, + "detectors_skipped": detectors_skipped, + "detector_summaries": detector_summaries, + "detector_vote_thresholds": { + name: round(value, 4) for name, value in vote_thresholds.items() + }, "threshold": float(threshold), "contamination": float(contamination), + "expected_anomaly_count": int(math.ceil(n_rows * contamination)), "flagged_count": flagged_count, + "flagged_fraction": round(float(flagged_count / n_rows), 4), + "consensus": { + "majority_detectors_required": majority_required, + "flagged_count": consensus_flagged_count, + "flagged_fraction": round(float(consensus_flagged_count / n_rows), 4), + }, "ensemble_summary": {k: round(v, 4) for k, v in summary.items()}, "top_rows": top_rows, "interpretation": ( - "Each detector emits a [0,1] anomaly score; the ensemble is the per-row mean " - "across all available detectors. Rows with ensemble >= " - f"{threshold} are flagged." + "Each available detector emits a normalized [0,1] anomaly score; the historical " + "ensemble remains their per-row mean. The configured threshold controls the main " + "flag, while detector agreement independently reports how many detectors place a row " + "in their contamination-adjusted anomaly tail. Top feature deviations are standardized " + "context, not causal explanations." ), } diff --git a/src/framevitals/api.py b/src/framevitals/api.py index 7b57be0..da8bb0f 100644 --- a/src/framevitals/api.py +++ b/src/framevitals/api.py @@ -1,181 +1,341 @@ +"""Backward-compatible public API facade. + +Historically this module contained a second eager implementation of most +FrameVitals operations. The canonical implementations now live in focused, +source-aware modules. Keeping this file as a thin lazy facade preserves imports +such as ``from framevitals.api import analyze`` without maintaining two engines +that can diverge in behavior, streaming support, or execution metadata. +""" + from __future__ import annotations from collections.abc import Mapping -from pathlib import Path -from typing import Any -from uuid import uuid4 - -import pandas as pd - -from framevitals.contracts import infer_contract as _infer_contract -from framevitals.contracts import validate_contract -from framevitals.drift_analysis import compare_datasets -from framevitals.loader import load_dataset -from framevitals.pipeline import run_full_analysis - -VALID_MODES = { - "quick", - "standard", - "deep", - "research", -} - -DataInput = str | Path | pd.DataFrame - - -def _validated_path(value: str | Path, *, label: str = "Dataset") -> Path: - path = Path(value) - if not path.exists(): - raise FileNotFoundError(f"{label} not found: {path}") - if not path.is_file(): - raise ValueError(f"Expected a file for {label.lower()}, got: {path}") - return path - - -def _load_input(value: DataInput, *, label: str) -> tuple[pd.DataFrame, str]: - if isinstance(value, pd.DataFrame): - if value.empty: - raise ValueError(f"{label} DataFrame is empty.") - return value.copy(), "" - - if isinstance(value, (str, Path)): - path = _validated_path(value, label=label) - df = load_dataset(path) - if df.empty: - raise ValueError(f"{label} dataset is empty: {path}") - return df, path.name - - raise TypeError( - f"{label} must be a pandas DataFrame or a path to a supported dataset." +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import pandas as pd + + from framevitals.cleaning_plan import CleaningPlan + from framevitals.planning import AnalysisPlan + from framevitals.quality_results import ( + CheckResult, + DriftResult, + GateResult, + ValidationResult, + ) + from framevitals.result import AnalysisResult, DiagnosticResult + + +DataInput = Any + + +def inspect_source(data: DataInput) -> dict[str, Any]: + """Inspect source metadata/capabilities without running analysis.""" + from framevitals.sources import inspect_source as _inspect_source + + return _inspect_source(data) + + +def profile(data: DataInput) -> DiagnosticResult: + """Profile a dataset through the canonical focused execution path.""" + from framevitals.focused import profile as _profile + + return _profile(data) + + +def roles(data: DataInput) -> DiagnosticResult: + """Infer column roles through the canonical focused execution path.""" + from framevitals.focused import roles as _roles + + return _roles(data) + + +def health(data: DataInput) -> DiagnosticResult: + """Calculate dataset health through the canonical focused execution path.""" + from framevitals.focused import health as _health + + return _health(data) + + +def ml_readiness(data: DataInput) -> DiagnosticResult: + """Calculate ML readiness through the canonical focused execution path.""" + from framevitals.focused import ml_readiness as _ml_readiness + + return _ml_readiness(data) + + +def quality( + data: DataInput, + *, + max_sample_rows: int = 5_000, + max_columns: int = 100, + max_missingness_columns: int = 25, +) -> DiagnosticResult: + """Run deterministic quality diagnostics through the focused engine.""" + from framevitals.focused import quality as _quality + + return _quality( + data, + max_sample_rows=max_sample_rows, + max_columns=max_columns, + max_missingness_columns=max_missingness_columns, ) +def statistics( + data: DataInput, + *, + max_pairs: int = 20, + mode: str = "standard", +) -> DiagnosticResult: + """Run bounded deep statistics through the focused engine.""" + from framevitals.focused import statistics as _statistics + + return _statistics(data, max_pairs=max_pairs, mode=mode) + + +def anomalies( + data: DataInput, + *, + contamination: float = 0.05, + threshold: float = 0.6, + max_columns: int = 30, + top_k: int = 25, + mode: str = "standard", +) -> DiagnosticResult: + """Run bounded anomaly diagnostics through the focused engine.""" + from framevitals.focused import anomalies as _anomalies + + return _anomalies( + data, + contamination=contamination, + threshold=threshold, + max_columns=max_columns, + top_k=top_k, + mode=mode, + ) + + +def relationships( + data: DataInput, + *, + max_sample_rows: int = 512, + projections: int = 64, + min_abs_correlation: float = 0.80, + max_candidate_pairs: int = 250_000, + max_edges_returned: int = 5_000, +) -> DiagnosticResult: + """Discover strong numeric relationships through the focused engine.""" + from framevitals.focused import relationships as _relationships + + return _relationships( + data, + max_sample_rows=max_sample_rows, + projections=projections, + min_abs_correlation=min_abs_correlation, + max_candidate_pairs=max_candidate_pairs, + max_edges_returned=max_edges_returned, + ) + + +def target_analysis(data: DataInput, *, target: str) -> DiagnosticResult: + """Run target diagnostics through the focused source-aware engine.""" + from framevitals.focused import target_analysis as _target_analysis + + return _target_analysis(data, target=target) + + def analyze( data: DataInput, *, target: str | None = None, - mode: str = "standard", - artifacts: bool = False, -) -> dict: - """Analyze a tabular dataset with FrameVitals. - - Parameters - ---------- - data: - A pandas DataFrame or a path to a CSV, TSV, Excel, or JSON dataset. - target: - Optional supervised-learning target column. - mode: - Analysis depth: ``quick``, ``standard``, ``deep``, or ``research``. - artifacts: - When ``True``, persist cleaned CSV/chart artifacts. The reusable Python - API defaults to ``False`` so analysis does not modify the filesystem. - - Returns - ------- - dict - Structured FrameVitals analysis results. - - Examples - -------- - >>> import pandas as pd - >>> import framevitals as fv - >>> df = pd.DataFrame({"age": [20, 30], "income": [30000, 50000]}) - >>> report = fv.analyze(df, mode="quick") - >>> print(report["health"]["overall_score"]) - """ - if mode not in VALID_MODES: - raise ValueError( - f"Invalid analysis mode '{mode}'. " - f"Choose from: {', '.join(sorted(VALID_MODES))}" - ) - - dataset_id = f"fv_{uuid4().hex[:12]}" - - if isinstance(data, pd.DataFrame): - if data.empty: - raise ValueError("Dataset DataFrame is empty.") - return run_full_analysis( - dataset_id=dataset_id, - dataframe=data, - original_filename="", - analysis_mode=mode, - target_column=target, - skip_ai=True, - write_artifacts=artifacts, - ) - - if isinstance(data, (str, Path)): - path = _validated_path(data) - return run_full_analysis( - dataset_id=dataset_id, - file_path=path, - original_filename=path.name, - analysis_mode=mode, - target_column=target, - skip_ai=True, - write_artifacts=artifacts, - ) - - raise TypeError( - "data must be a pandas DataFrame or a path to a supported dataset." + mode: str | None = None, + artifacts: bool | None = None, + workers: int | None = None, + preset: str | None = None, + config: Any = None, + disabled_modules: list[str] | tuple[str, ...] | None = None, +) -> AnalysisResult: + """Analyze a dataset through the canonical source-aware dispatcher.""" + from framevitals.analysis_api import analyze as _analyze + + return _analyze( + data, + target=target, + mode=mode, + artifacts=artifacts, + workers=workers, + preset=preset, + config=config, + disabled_modules=disabled_modules, ) +def plan( + data: DataInput, + *, + target: str | None = None, + mode: str | None = None, + workers: int | None = None, + preset: str | None = None, + config: Any = None, + disabled_modules: list[str] | tuple[str, ...] | None = None, +) -> AnalysisPlan: + """Preview analysis execution through the canonical planning API.""" + from framevitals.planning_api import plan as _plan + + return _plan( + data, + target=target, + mode=mode, + workers=workers, + preset=preset, + config=config, + disabled_modules=disabled_modules, + ) + + +def plan_cleaning(data: DataInput) -> CleaningPlan: + """Infer a conservative cleaning plan through the operations layer.""" + from framevitals.operations import plan_cleaning as _plan_cleaning + + return _plan_cleaning(data) + + +def clean( + data: DataInput, + *, + plan: Mapping[str, Any] | None = None, +) -> pd.DataFrame: + """Return an explicitly cleaned copy through the operations layer.""" + from framevitals.operations import clean as _clean + + return _clean(data, plan=plan) + + def compare( reference: DataInput, current: DataInput, *, columns: list[str] | None = None, max_columns: int = 30, -) -> dict: - """Compare reference and current datasets for distribution drift. - - Both inputs may independently be pandas DataFrames or supported dataset - paths. Numeric features use PSI, KS statistics, and standardized mean - shift; categorical features use PSI and chi-square diagnostics. - """ - if max_columns < 1: - raise ValueError("max_columns must be at least 1.") - - ref_df, ref_name = _load_input(reference, label="Reference") - cur_df, cur_name = _load_input(current, label="Current") - - result = compare_datasets( - ref_df, - cur_df, +) -> DriftResult: + """Compare reference/current data through the source-aware drift path.""" + from framevitals.operations import compare as _compare + + return _compare( + reference, + current, columns=columns, max_columns=max_columns, ) - result["reference_name"] = ref_name - result["current_name"] = cur_name - return result -def infer_contract(data: DataInput) -> dict[str, Any]: - """Infer a JSON-serializable data contract from a reference dataset. +def infer_contract( + data: DataInput, + *, + numeric_tolerance: float = 0.05, + max_categories: int = 20, + null_fraction_tolerance: float = 0.05, + infer_unique: bool = True, + min_unique_rows: int = 20, + allow_extra_columns: bool = False, +) -> dict[str, Any]: + """Infer a reusable data contract through the operations layer.""" + from framevitals.operations import infer_contract as _infer_contract - The result can be saved with :mod:`json` and passed to :func:`validate` - when checking later datasets in a pipeline or CI job. - """ - dataframe, source_name = _load_input(data, label="Reference") - contract = _infer_contract(dataframe) - contract["reference_name"] = source_name - return contract + return _infer_contract( + data, + numeric_tolerance=numeric_tolerance, + max_categories=max_categories, + null_fraction_tolerance=null_fraction_tolerance, + infer_unique=infer_unique, + min_unique_rows=min_unique_rows, + allow_extra_columns=allow_extra_columns, + ) -def validate( - data: DataInput, - contract: Mapping[str, Any], -) -> dict[str, Any]: - """Validate a dataset against an inferred or explicit data contract. - - Contract failures are returned as structured findings rather than raised, - allowing callers to decide whether warnings or errors should block a job. - Invalid contract definitions and unreadable datasets still raise clear - exceptions. - """ - dataframe, source_name = _load_input(data, label="Dataset") - result = validate_contract(dataframe, contract) - result["dataset_name"] = source_name - return result +def validate(data: DataInput, contract: Mapping[str, Any]) -> ValidationResult: + """Validate a dataset exactly through the operations layer.""" + from framevitals.operations import validate as _validate + + return _validate(data, contract) + + +def check( + name: str | None = None, + *, + severity: str = "error", + description: str | None = None, +): + """Decorate a predicate as a reusable custom data check.""" + from framevitals.checks import check as _check + + return _check(name, severity=severity, description=description) + + +def run_checks(data: DataInput, checks: Any) -> CheckResult: + """Run exact custom checks through the canonical check engine.""" + from framevitals.checks import run_checks as _run_checks + + return _run_checks(data, checks) + + +def discover_checks(*, group: str = "framevitals.checks"): + """Explicitly load installed third-party check plugins.""" + from framevitals.plugins import discover_checks as _discover_checks + + return _discover_checks(group=group) + + +def gate( + current: DataInput, + *, + reference: DataInput | None = None, + contract: Mapping[str, Any] | None = None, + custom_checks: Any = None, + columns: list[str] | None = None, + max_columns: int = 30, + drift_warn_on: str = "moderate", + drift_fail_on: str = "severe", + fail_on_validation_warning: bool = False, +) -> GateResult: + """Run the canonical contract/custom/drift quality gate.""" + from framevitals.operations import gate as _gate + + return _gate( + current, + reference=reference, + contract=contract, + custom_checks=custom_checks, + columns=columns, + max_columns=max_columns, + drift_warn_on=drift_warn_on, + drift_fail_on=drift_fail_on, + fail_on_validation_warning=fail_on_validation_warning, + ) + + +__all__ = [ + "inspect_source", + "profile", + "roles", + "health", + "ml_readiness", + "quality", + "statistics", + "anomalies", + "relationships", + "target_analysis", + "analyze", + "plan", + "plan_cleaning", + "clean", + "compare", + "infer_contract", + "validate", + "check", + "run_checks", + "discover_checks", + "gate", +] diff --git a/src/framevitals/backends.py b/src/framevitals/backends.py new file mode 100644 index 0000000..282faa8 --- /dev/null +++ b/src/framevitals/backends.py @@ -0,0 +1,218 @@ +"""Backend routing for FrameVitals native and NumPy kernels. + +The router keeps optional native/GPU dependencies behind lazy imports. Compiled +FrameVitals native kernels are preferred when available; NumPy remains the +portable reference/fallback backend for compatible primitives. +""" + +from __future__ import annotations + +from importlib import import_module +from importlib.util import find_spec +import os +from typing import Any, Literal + +import numpy as np +import pandas as pd + + +BackendName = Literal["auto", "numpy", "rust"] +_VALID_BACKENDS = {"auto", "numpy", "rust"} + + +def native_available() -> bool: + """Return whether the optional FrameVitals native extension is importable.""" + return find_spec("framevitals._native") is not None + + +def resolve_numeric_backend(requested: str | None = None) -> Literal["numpy", "rust"]: + """Resolve the backend for exact/streaming primitives. + + ``FRAMEVITALS_BACKEND`` may set ``auto``, ``numpy`` or ``rust``. Explicit + function arguments take precedence. Requesting Rust without the extension + installed is an error; ``auto`` always falls back safely to NumPy. + """ + value = (requested or os.getenv("FRAMEVITALS_BACKEND", "auto")).strip().lower() + if value not in _VALID_BACKENDS: + raise ValueError( + "Unknown FrameVitals numeric backend " + f"'{value}'. Choose from: auto, numpy, rust." + ) + if value == "numpy": + return "numpy" + if value == "rust": + if not native_available(): + raise RuntimeError( + "The Rust backend was requested but framevitals._native is not installed." + ) + return "rust" + return "rust" if native_available() else "numpy" + + +def _float64_array(values: pd.Series | np.ndarray | list[Any]) -> np.ndarray: + if isinstance(values, pd.Series): + array = pd.to_numeric(values, errors="coerce").to_numpy( + dtype="float64", + na_value=np.nan, + ) + else: + array = np.asarray(values, dtype=np.float64) + if array.ndim != 1: + raise ValueError("FrameVitals numeric kernels require one-dimensional input.") + return np.ascontiguousarray(array, dtype=np.float64) + + +def _bias_corrected_shape( + count: int, + m2: float, + m3: float, + m4: float, +) -> tuple[float | None, float | None]: + """Return pandas/SciPy-compatible sample skewness and excess kurtosis.""" + if count < 3 or m2 <= 0.0: + skewness = None + else: + n = float(count) + population_skew = np.sqrt(n) * m3 / (m2 ** 1.5) + skewness = float(np.sqrt(n * (n - 1.0)) / (n - 2.0) * population_skew) + + if count < 4 or m2 <= 0.0: + kurtosis = None + else: + n = float(count) + population_excess = n * m4 / (m2 * m2) - 3.0 + kurtosis = float( + (n - 1.0) + / ((n - 2.0) * (n - 3.0)) + * ((n + 1.0) * population_excess + 6.0) + ) + return skewness, kurtosis + + +def _numpy_numeric_state(array: np.ndarray) -> dict[str, Any]: + missing = int(np.isnan(array).sum()) + infinite = int(np.isinf(array).sum()) + finite = array[np.isfinite(array)] + if finite.size == 0: + return { + "backend": "numpy", + "observations": int(array.size), + "count": 0, + "missing": missing, + "infinite": infinite, + "mean": None, + "variance": None, + "std": None, + "skewness": None, + "kurtosis": None, + "m2": 0.0, + "m3": 0.0, + "m4": 0.0, + "minimum": None, + "maximum": None, + } + + mean = float(finite.mean()) + centered = finite - mean + centered2 = centered * centered + m2 = float(np.sum(centered2)) + m3 = float(np.dot(centered2, centered)) + m4 = float(np.dot(centered2, centered2)) + variance = m2 / (finite.size - 1) if finite.size >= 2 else None + skewness, kurtosis = _bias_corrected_shape(int(finite.size), m2, m3, m4) + return { + "backend": "numpy", + "observations": int(array.size), + "count": int(finite.size), + "missing": missing, + "infinite": infinite, + "mean": mean, + "variance": variance, + "std": float(np.sqrt(variance)) if variance is not None else None, + "skewness": skewness, + "kurtosis": kurtosis, + "m2": m2, + "m3": m3, + "m4": m4, + "minimum": float(finite.min()), + "maximum": float(finite.max()), + } + + +def numeric_state( + values: pd.Series | np.ndarray | list[Any], + *, + backend: BackendName | str | None = None, +) -> dict[str, Any]: + """Calculate exact mergeable moments through fourth order.""" + array = _float64_array(values) + selected = resolve_numeric_backend(backend) + if selected == "rust": + native = import_module("framevitals._native") + payload = native.numeric_state_f64(array) + return dict(payload) + return _numpy_numeric_state(array) + + +def numeric_profile( + values: pd.Series | np.ndarray | list[Any], + *, + backend: BackendName | str | None = None, + stream_id: int = 0, +) -> dict[str, Any]: + """Run the fused native numeric profile when available.""" + array = _float64_array(values) + selected = resolve_numeric_backend(backend) + if selected == "rust": + native = import_module("framevitals._native") + payload = dict(native.numeric_profile_f64(array, stream_id=int(stream_id))) + payload["sketches_available"] = True + return payload + + payload = _numpy_numeric_state(array) + payload["sketches_available"] = False + return payload + + +def create_numeric_accumulator(*, stream_id: int = 0): + """Create a persistent native numeric accumulator for multi-batch scans.""" + if resolve_numeric_backend() != "rust": + return None + native = import_module("framevitals._native") + return native.NumericAccumulator(stream_id=int(stream_id)) + + +def create_arrow_batch_profile_accumulator(): + """Create the zero-copy native Arrow RecordBatch profiler when available. + + Returning ``None`` for an older native extension preserves compatibility + with installations that only expose the per-column float64 accumulator. + """ + if resolve_numeric_backend() != "rust": + return None + native = import_module("framevitals._native") + accumulator = getattr(native, "ArrowBatchProfileAccumulator", None) + return accumulator() if accumulator is not None else None + + +def create_string_accumulator(): + """Create a persistent native UTF-8 sketch accumulator when available.""" + if resolve_numeric_backend() != "rust": + return None + native = import_module("framevitals._native") + accumulator = getattr(native, "StringAccumulator", None) + return accumulator() if accumulator is not None else None + + +def backend_status() -> dict[str, Any]: + """Return lightweight backend availability without importing native code.""" + has_native = native_available() + eligible = ["numpy"] + if has_native: + eligible.append("rust") + return { + "selected": resolve_numeric_backend("auto"), + "native_available": has_native, + "environment_override": os.getenv("FRAMEVITALS_BACKEND"), + "eligible": eligible, + } diff --git a/src/framevitals/budgeted_analysis.py b/src/framevitals/budgeted_analysis.py new file mode 100644 index 0000000..d7c8c61 --- /dev/null +++ b/src/framevitals/budgeted_analysis.py @@ -0,0 +1,267 @@ +"""Resource-bounded adapters around expensive analysis modules. + +These adapters make legacy analyses safe on large inputs while FrameVitals moves +more work into streaming/native kernels. Sampling and adaptive column selection +are deterministic and are always disclosed in returned execution metadata. +""" + +from __future__ import annotations + +from typing import Any + +import pandas as pd + +from framevitals.anomaly_ensemble import detect_anomalies_ensemble +from framevitals.deep_statistics_v2 import run_deep_statistics_v2 +from framevitals.deep_triage import triage_deep_columns +from framevitals.execution import ExecutionBudget, deterministic_sample_frame +from framevitals.fast_anomaly import fast_anomaly_scan +from framevitals.fast_deep_statistics import run_fast_deep_statistics_v2 +from framevitals.neural_anomaly import neural_reconstruction_anomalies +from framevitals.provenance import normalize_execution +from framevitals.stream_change import scan_ordered_mean_shift +from framevitals.time_series import detect_and_analyze_time_series + + +_RESEARCH_BCA_MAX_ROWS = 2_000 + + +def _attach_execution( + payload: dict[str, Any], + *, + budget: ExecutionBudget, + sampling: dict[str, Any], + scope: str, +) -> dict[str, Any]: + result = dict(payload) + result["execution"] = normalize_execution( + { + "scope": scope, + "scale_class": budget.scale_class, + **sampling, + }, + method=scope, + full_materialization=False, + ) + return result + + +def run_budgeted_deep_statistics( + dataframe: pd.DataFrame, + *, + budget: ExecutionBudget, + max_pairs: int | None = None, +) -> dict[str, Any]: + """Run deep statistics on a bounded, adaptively selected diagnostic view. + + Research mode keeps BCa bootstrap semantics for genuinely small diagnostic + samples, where resampling cost is bounded and finite-sample refinement can + be useful. Once the diagnostic sample is larger, deterministic Student-t + mean intervals and distribution-free median order-statistic intervals are + used instead. The rest of the Research statistical battery is unchanged. + """ + sample_limit = max( + 20, + min( + budget.deep_statistics_sample_rows, + budget.bootstrap_sample_rows, + ), + ) + sample_limit = min(sample_limit, max(len(dataframe), 1)) + work, sampling = deterministic_sample_frame(dataframe, sample_limit) + + triage = triage_deep_columns(work, mode=budget.mode) + selected_columns = list(triage.selected_columns) + diagnostic_view = ( + work.loc[:, selected_columns] + if selected_columns + else pd.DataFrame(index=work.index) + ) + + pair_budget = ( + budget.relationship_pair_budget + if max_pairs is None + else min(int(max_pairs), budget.relationship_pair_budget) + ) + if pair_budget < 1: + raise ValueError("max_pairs must be at least 1.") + + use_research_bca = ( + budget.mode == "research" and len(diagnostic_view) <= _RESEARCH_BCA_MAX_ROWS + ) + if use_research_bca: + payload = run_deep_statistics_v2(diagnostic_view, max_pairs=pair_budget) + inference_strategy = "bca_bootstrap_small_sample" + else: + payload = run_fast_deep_statistics_v2( + diagnostic_view, + max_pairs=pair_budget, + ) + inference_strategy = ( + "adaptive_large_sample_closed_form_and_order_statistics" + if budget.mode == "research" + else "closed_form_and_order_statistics" + ) + + triage_payload = triage.to_dict() + payload["column_triage"] = triage_payload + + sampling = { + **sampling, + "reason": ( + "Deep statistics use a bounded row view plus adaptive column triage. " + "Research mode retains BCa bootstrap on small diagnostic samples but " + "switches to deterministic large-sample intervals above the BCa cost " + "threshold; other modes use the deterministic interval path directly." + ), + "pair_budget": int(pair_budget), + "column_triage": triage_payload, + "source_columns": int(dataframe.shape[1]), + "diagnostic_columns": int(len(selected_columns)), + "adaptive_strategy": "column_interest_triage", + "inference_strategy": inference_strategy, + "research_bca_max_rows": int(_RESEARCH_BCA_MAX_ROWS), + } + return _attach_execution( + payload, + budget=budget, + sampling=sampling, + scope="bounded_deep_statistics", + ) + + +def run_budgeted_anomalies( + dataframe: pd.DataFrame, + *, + budget: ExecutionBudget, + contamination: float = 0.05, + threshold: float = 0.6, + max_columns: int = 30, + top_k: int = 25, +) -> dict[str, Any]: + """Run fast screening in standard/deep and full confirmation in research.""" + sample_limit = max(20, min(budget.anomaly_sample_rows, max(len(dataframe), 1))) + work, sampling = deterministic_sample_frame(dataframe, sample_limit) + + if budget.mode == "research": + payload = detect_anomalies_ensemble( + work, + contamination=contamination, + threshold=threshold, + max_columns=max_columns, + top_k=top_k, + ) + anomaly_strategy = "classical_ensemble_plus_neural_reconstruction" + try: + payload["neural_reconstruction"] = neural_reconstruction_anomalies( + work, + max_rows=min(3_000, max(len(work), 20)), + max_columns=min(max_columns, 24), + max_iter=35, + top_k=top_k, + ) + except Exception as exc: # neural diagnostics must fail soft + payload["neural_reconstruction"] = { + "available": False, + "reason": f"{type(exc).__name__}: {exc}", + } + else: + payload = fast_anomaly_scan( + work, + contamination=contamination, + threshold=threshold, + max_columns=min(max_columns, 24), + projections=12, + top_k=top_k, + ) + anomaly_strategy = "fast_robust_random_projection" + + sampling = { + **sampling, + "reason": ( + "Research mode confirms anomalies with the heavier classical ensemble and " + "a bounded neural reconstruction detector." + if budget.mode == "research" + else "Standard/deep modes use vectorized robust and random-projection anomaly screening." + ), + "coverage": "sample" if sampling["sampled"] else "full", + "anomaly_strategy": anomaly_strategy, + "adaptive_strategy": anomaly_strategy, + "neural_reconstruction_enabled": budget.mode == "research", + } + return _attach_execution( + payload, + budget=budget, + sampling=sampling, + scope="bounded_anomaly_detection", + ) + + +def _attach_time_series_change_scan( + payload: dict[str, Any], + work: pd.DataFrame, +) -> bool: + """Attach a cheap ordered mean-shift scan when time-series detection succeeded.""" + if not payload.get("available"): + return False + date_column = payload.get("detected_date_column") + numeric_column = payload.get("numeric_column") + if not isinstance(date_column, str) or not isinstance(numeric_column, str): + return False + if date_column not in work.columns or numeric_column not in work.columns: + return False + + parsed_dates = pd.to_datetime(work[date_column], errors="coerce", format="mixed") + ordered = pd.DataFrame({ + "date": parsed_dates, + "value": pd.to_numeric(work[numeric_column], errors="coerce"), + }).dropna() + if ordered.empty: + return False + ordered = ordered.sort_values("date") + payload["mean_shift"] = scan_ordered_mean_shift( + ordered["value"], + windows=24, + threshold=8.0, + min_updates=8, + ) + return True + + +def run_budgeted_time_series( + dataframe: pd.DataFrame, + *, + budget: ExecutionBudget, + target_column: str | None = None, +) -> dict[str, Any]: + """Run bounded ordered time-series diagnostics plus change detection.""" + sample_limit = max(30, min(budget.time_series_sample_rows, max(len(dataframe), 1))) + work, sampling = deterministic_sample_frame( + dataframe, + sample_limit, + preserve_order=True, + ) + payload = detect_and_analyze_time_series(work, target_column=target_column) + change_detection_enabled = _attach_time_series_change_scan(payload, work) + sampling = { + **sampling, + "reason": ( + "Time-series diagnostics use an order-preserving bounded view to avoid " + "unbounded date parsing, stationarity, PACF, STL, and forecasting work. " + "Detected numeric series also receive a bounded Page-Hinkley mean-shift scan." + if sampling["sampled"] + else ( + "Full input fits within the time-series execution budget; detected numeric " + "series also receive a bounded Page-Hinkley mean-shift scan." + ) + ), + "temporal_order_preserved": True, + "mean_shift_detection_enabled": bool(change_detection_enabled), + "adaptive_strategy": "ordered_page_hinkley_mean_shift", + } + return _attach_execution( + payload, + budget=budget, + sampling=sampling, + scope="bounded_time_series", + ) diff --git a/src/framevitals/checks.py b/src/framevitals/checks.py new file mode 100644 index 0000000..f8b1daf --- /dev/null +++ b/src/framevitals/checks.py @@ -0,0 +1,213 @@ +"""Extensible user-defined checks for FrameVitals quality gates.""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Literal, cast + +import numpy as np +import pandas as pd + +from framevitals.provenance import execution_provenance, load_fully_materializes +from framevitals.quality_results import CheckResult +from framevitals.sources import resolve_source + + +CheckSeverity = Literal["warning", "error"] +CheckFunction = Callable[[pd.DataFrame], bool | np.bool_ | Mapping[str, Any]] +DataInput = Any + + +def _slug(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_") + return slug or "check" + + +def _validate_severity(value: str) -> CheckSeverity: + if value not in {"warning", "error"}: + raise ValueError("check severity must be 'warning' or 'error'.") + return cast(CheckSeverity, value) + + +@dataclass(frozen=True, slots=True) +class DataCheck: + """Named user-defined data check with gate severity metadata.""" + + name: str + function: CheckFunction + severity: CheckSeverity = "error" + description: str | None = None + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("check name must not be empty.") + if not callable(self.function): + raise TypeError("check function must be callable.") + _validate_severity(self.severity) + + def __call__(self, dataframe: pd.DataFrame) -> bool | np.bool_ | Mapping[str, Any]: + return self.function(dataframe) + + +def check( + name: str | None = None, + *, + severity: CheckSeverity = "error", + description: str | None = None, +): + """Decorate a DataFrame predicate as a reusable :class:`DataCheck`. + + A check can return a boolean or a mapping containing at least ``passed``. + Optional mapping keys include ``message`` and ``details``. + """ + resolved_severity = _validate_severity(severity) + + def decorator(function: CheckFunction) -> DataCheck: + resolved_name = (name or getattr(function, "__name__", "check")).strip() + return DataCheck( + name=resolved_name, + function=function, + severity=resolved_severity, + description=description, + ) + + return decorator + + +def _normalize_check(value: DataCheck | CheckFunction) -> DataCheck: + if isinstance(value, DataCheck): + return value + if not callable(value): + raise TypeError("custom checks must be DataCheck instances or callables.") + return DataCheck( + name=getattr(value, "__name__", "check"), + function=value, + ) + + +def _normalize_outcome( + definition: DataCheck, + raw: bool | np.bool_ | Mapping[str, Any], +) -> dict[str, Any]: + if isinstance(raw, (bool, np.bool_)): + passed = bool(raw) + message = ( + f"{definition.name} passed." + if passed + else f"{definition.name} failed." + ) + details: Any = None + elif isinstance(raw, Mapping): + if "passed" not in raw: + raise ValueError( + f"Custom check {definition.name!r} returned a mapping without 'passed'." + ) + passed = bool(raw["passed"]) + default_message = ( + f"{definition.name} passed." + if passed + else f"{definition.name} failed." + ) + message = str(raw.get("message") or default_message) + details = raw.get("details") + else: + raise TypeError( + f"Custom check {definition.name!r} must return bool or a mapping." + ) + + return { + "name": definition.name, + "code": f"custom.{_slug(definition.name)}", + "passed": passed, + "severity": definition.severity, + "description": definition.description, + "message": message, + "details": details, + } + + +def run_checks( + data: DataInput, + checks: Sequence[DataCheck | CheckFunction], +) -> CheckResult: + """Run user-defined checks exactly against a materialized DataFrame. + + Custom Python callables can inspect arbitrary row-level relationships, so + FrameVitals does not silently sample their input. Non-pandas sources are + materialized intentionally and that decision is disclosed in ``execution``. + """ + definitions = [_normalize_check(value) for value in checks] + if not definitions: + raise ValueError("run_checks requires at least one custom check.") + + source = resolve_source(data) + metadata = source.inspect() + dataframe = source.load() + + results: list[dict[str, Any]] = [] + for definition in definitions: + try: + raw = definition(dataframe.copy()) + outcome = _normalize_outcome(definition, raw) + outcome["execution_error"] = None + except Exception as exc: # user-defined check failures become structured results + outcome = { + "name": definition.name, + "code": f"custom.{_slug(definition.name)}", + "passed": False, + "severity": "error", + "description": definition.description, + "message": f"Custom check raised {type(exc).__name__}: {exc}", + "details": None, + "execution_error": f"{type(exc).__name__}: {exc}", + } + results.append(outcome) + + failures = [result for result in results if not result["passed"]] + error_failures = [ + result for result in failures if result.get("severity") == "error" + ] + warning_failures = [ + result for result in failures if result.get("severity") == "warning" + ] + status = "fail" if error_failures else "warn" if warning_failures else "pass" + + findings = [ + { + "code": result["code"], + "severity": result["severity"], + "title": result["name"], + "message": result["message"], + "details": result.get("details"), + } + for result in failures + ] + + execution = execution_provenance( + "exact_custom_checks", + full_materialization=load_fully_materializes(metadata), + source=metadata.to_dict(), + sampled=False, + source_rows=metadata.rows, + source_columns=metadata.columns, + reason=( + "Arbitrary custom Python checks run on the complete DataFrame; " + "FrameVitals does not silently sample user-defined invariants." + ), + ) + + return CheckResult({ + "status": status, + "passed": status != "fail", + "results": results, + "findings": findings, + "summary": { + "checks": len(results), + "passed": len(results) - len(failures), + "warnings": len(warning_failures), + "errors": len(error_failures), + }, + "execution": execution, + }) diff --git a/src/framevitals/cleaner.py b/src/framevitals/cleaner.py index 352dfde..b64ca0f 100644 --- a/src/framevitals/cleaner.py +++ b/src/framevitals/cleaner.py @@ -1,7 +1,7 @@ from pathlib import Path -import pandas as pd +from framevitals.cleaning_plan import apply_cleaning_plan, infer_cleaning_plan from framevitals.health_score import calculate_health_score from framevitals.profiler import build_profile from framevitals.security import sanitize_csv_value @@ -9,6 +9,37 @@ CLEANED_DIR = Path("cleaned") +def _legacy_action_view(action: dict) -> dict: + """Keep the existing cleaner action payload stable for current callers.""" + action_type = action.get("type") + affected = int(action.get("affected", 0)) + column = action.get("column") + + if action_type == "remove_duplicates": + return { + "action": "Remove duplicates", + "details": f"Removed {affected} duplicate rows.", + "risk": "Low", + } + if action_type == "fill_numeric_missing": + return { + "action": "Fill numeric missing values", + "details": f"Filled {affected} missing values in '{column}' using median.", + "risk": "Medium", + } + if action_type == "fill_categorical_missing": + return { + "action": "Fill categorical missing values", + "details": f"Filled {affected} missing values in '{column}' using mode.", + "risk": "Medium", + } + return { + "action": str(action_type or "Cleaning action"), + "details": str(action.get("description") or ""), + "risk": str(action.get("risk") or "Unknown").title(), + } + + def create_cleaned_dataset( dataset_id, df, @@ -20,66 +51,26 @@ def create_cleaned_dataset( ): """Build a conservative cleaned copy and optionally persist it as CSV. - Library callers can set ``write_output=False`` to keep analysis free of - filesystem side effects. Application callers retain the historical - behavior by leaving ``write_output`` enabled. - - ``before_profile`` and ``before_health`` are optional cached values. The - main pipeline supplies them to avoid profiling/scoring the original data a - second time; standalone callers can omit them with unchanged behavior. + The implementation now uses the public cleaning-plan primitives internally, + but preserves the historical ``actions``/health/count payload for backward + compatibility. The structured plan is returned additionally under ``plan``. """ - cleaned = df.copy() - actions = [] - if before_profile is None: before_profile = build_profile(df) if before_health is None: before_health = calculate_health_score(df, before_profile) - cached_duplicates = before_profile.get("duplicate_rows") - duplicate_count = int( - cached_duplicates if cached_duplicates is not None else df.duplicated().sum() - ) + plan = infer_cleaning_plan(df, profile=before_profile) + cleaned = apply_cleaning_plan(df, plan, copy=True) + actions = [_legacy_action_view(action) for action in plan.actions] + + after_profile = build_profile(cleaned) + after_health = calculate_health_score(cleaned, after_profile) missing_before = sum( int(value) for value in before_profile.get("missing_counts", {}).values() if value is not None ) - - if duplicate_count: - cleaned = cleaned.drop_duplicates() - actions.append({ - "action": "Remove duplicates", - "details": f"Removed {duplicate_count} duplicate rows.", - "risk": "Low", - }) - - missing_counts = cleaned.isna().sum() - for col, missing_value in missing_counts.items(): - missing = int(missing_value) - if missing == 0: - continue - - if pd.api.types.is_numeric_dtype(cleaned[col]): - value = cleaned[col].median() - cleaned[col] = cleaned[col].fillna(value) - actions.append({ - "action": "Fill numeric missing values", - "details": f"Filled {missing} missing values in '{col}' using median.", - "risk": "Medium", - }) - else: - mode = cleaned[col].mode(dropna=True) - value = mode.iloc[0] if len(mode) else "Unknown" - cleaned[col] = cleaned[col].fillna(value) - actions.append({ - "action": "Fill categorical missing values", - "details": f"Filled {missing} missing values in '{col}' using mode.", - "risk": "Medium", - }) - - after_profile = build_profile(cleaned) - after_health = calculate_health_score(cleaned, after_profile) missing_after = sum( int(value) for value in after_profile.get("missing_counts", {}).values() @@ -96,11 +87,12 @@ def create_cleaned_dataset( return { "actions": actions, + "plan": dict(plan), "before_health": before_health, "after_health": after_health, "output_path": str(output_path) if output_path is not None else None, "missing_before": int(missing_before), "missing_after": int(missing_after), - "duplicates_before": duplicate_count, + "duplicates_before": int(plan.get("duplicates_to_remove", 0)), "duplicates_after": int(after_profile.get("duplicate_rows", 0)), } diff --git a/src/framevitals/cleaning_plan.py b/src/framevitals/cleaning_plan.py new file mode 100644 index 0000000..ee486f9 --- /dev/null +++ b/src/framevitals/cleaning_plan.py @@ -0,0 +1,242 @@ +"""Explicit, conservative cleaning plans for FrameVitals datasets. + +Planning never mutates user data. Applying a plan returns a copy by default and +supports only the conservative operations already used by FrameVitals' internal +cleaner: duplicate removal and missing-value imputation by median/mode. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +import numpy as np +import pandas as pd + +from framevitals.health_score import calculate_health_score +from framevitals.profiler import build_profile + + +CLEANING_PLAN_SCHEMA_VERSION = "1" + + +def _python_scalar(value: Any) -> Any: + if isinstance(value, np.generic): + return value.item() + return value + + +class CleaningPlan(dict): + """Dict-compatible, inspectable cleaning plan.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + @property + def actions(self) -> list[dict[str, Any]]: + value = self.get("actions", []) + return value if isinstance(value, list) else [] + + def summary(self) -> dict[str, Any]: + risk_counts: dict[str, int] = {} + for action in self.actions: + risk = str(action.get("risk") or "unknown").lower() + risk_counts[risk] = risk_counts.get(risk, 0) + 1 + return { + "action_count": len(self.actions), + "risk_counts": risk_counts, + "duplicates_to_remove": self.get("duplicates_to_remove", 0), + "missing_values_to_fill": self.get("missing_values_to_fill", 0), + } + + def to_json( + self, + destination: str | Path | None = None, + *, + indent: int = 2, + ) -> str | Path: + rendered = json.dumps(dict(self), indent=indent, default=str) + if destination is None: + return rendered + path = Path(destination) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered + "\n", encoding="utf-8") + return path + + def apply(self, dataframe: pd.DataFrame, *, copy: bool = True) -> pd.DataFrame: + return apply_cleaning_plan(dataframe, self, copy=copy) + + def simulate( + self, + dataframe: pd.DataFrame, + *, + before_profile: dict | None = None, + before_health: dict | None = None, + ) -> dict[str, Any]: + return simulate_cleaning_plan( + dataframe, + self, + before_profile=before_profile, + before_health=before_health, + ) + + +def infer_cleaning_plan( + dataframe: pd.DataFrame, + *, + profile: dict | None = None, +) -> CleaningPlan: + """Infer the same conservative operations used by the internal cleaner.""" + if not isinstance(dataframe, pd.DataFrame): + raise TypeError("dataframe must be a pandas DataFrame") + if dataframe.empty: + raise ValueError("Cannot infer a cleaning plan for an empty DataFrame.") + + if profile is None: + profile = build_profile(dataframe) + + duplicate_count = int(profile.get("duplicate_rows", dataframe.duplicated().sum())) + working = dataframe.drop_duplicates() if duplicate_count else dataframe + actions: list[dict[str, Any]] = [] + + if duplicate_count: + actions.append({ + "id": "remove_duplicates", + "type": "remove_duplicates", + "column": None, + "strategy": "drop_duplicate_rows", + "affected": duplicate_count, + "risk": "low", + "description": f"Remove {duplicate_count} duplicate rows.", + }) + + missing_values_to_fill = 0 + missing_counts = working.isna().sum() + for column, missing_value in missing_counts.items(): + missing = int(missing_value) + if missing == 0: + continue + + missing_values_to_fill += missing + if pd.api.types.is_numeric_dtype(working[column]): + fill_value = working[column].median() + strategy = "median" + action_type = "fill_numeric_missing" + description = ( + f"Fill {missing} missing values in '{column}' using the median." + ) + else: + mode = working[column].mode(dropna=True) + fill_value = mode.iloc[0] if len(mode) else "Unknown" + strategy = "mode" if len(mode) else "constant" + action_type = "fill_categorical_missing" + description = ( + f"Fill {missing} missing values in '{column}' using " + f"{'the mode' if len(mode) else 'a fallback value'}." + ) + + actions.append({ + "id": f"{action_type}:{column}", + "type": action_type, + "column": str(column), + "strategy": strategy, + "fill_value": _python_scalar(fill_value), + "affected": missing, + "risk": "medium", + "description": description, + }) + + return CleaningPlan({ + "schema_version": CLEANING_PLAN_SCHEMA_VERSION, + "actions": actions, + "duplicates_to_remove": duplicate_count, + "missing_values_to_fill": missing_values_to_fill, + }) + + +def apply_cleaning_plan( + dataframe: pd.DataFrame, + plan: Mapping[str, Any], + *, + copy: bool = True, +) -> pd.DataFrame: + """Apply a validated FrameVitals cleaning plan and return the result.""" + if not isinstance(dataframe, pd.DataFrame): + raise TypeError("dataframe must be a pandas DataFrame") + if not isinstance(plan, Mapping): + raise TypeError("plan must be a cleaning-plan mapping") + if plan.get("schema_version") != CLEANING_PLAN_SCHEMA_VERSION: + raise ValueError( + "Unsupported cleaning plan schema version: " + f"{plan.get('schema_version')!r}" + ) + + actions = plan.get("actions", []) + if not isinstance(actions, list): + raise ValueError("Cleaning plan actions must be a list.") + + cleaned = dataframe.copy() if copy else dataframe + for action in actions: + if not isinstance(action, Mapping): + raise ValueError("Each cleaning plan action must be an object.") + + action_type = action.get("type") + if action_type == "remove_duplicates": + cleaned.drop_duplicates(inplace=True) + continue + + if action_type in {"fill_numeric_missing", "fill_categorical_missing"}: + column = action.get("column") + if not isinstance(column, str) or column not in cleaned.columns: + raise ValueError(f"Cleaning plan column not found: {column!r}") + cleaned[column] = cleaned[column].fillna(action.get("fill_value")) + continue + + raise ValueError(f"Unsupported cleaning action type: {action_type!r}") + + return cleaned + + +def simulate_cleaning_plan( + dataframe: pd.DataFrame, + plan: Mapping[str, Any], + *, + before_profile: dict | None = None, + before_health: dict | None = None, +) -> dict[str, Any]: + """Apply a plan to a copy and report expected quality/shape changes.""" + if before_profile is None: + before_profile = build_profile(dataframe) + if before_health is None: + before_health = calculate_health_score(dataframe, before_profile) + + cleaned = apply_cleaning_plan(dataframe, plan, copy=True) + after_profile = build_profile(cleaned) + after_health = calculate_health_score(cleaned, after_profile) + + before_shape = before_profile.get("shape", {}) + after_shape = after_profile.get("shape", {}) + before_score = before_health.get("overall_score") + after_score = after_health.get("overall_score") + + health_delta = None + if isinstance(before_score, (int, float)) and isinstance(after_score, (int, float)): + health_delta = round(float(after_score) - float(before_score), 4) + + return { + "before_shape": dict(before_shape), + "after_shape": dict(after_shape), + "rows_removed": int(before_shape.get("rows", len(dataframe))) + - int(after_shape.get("rows", len(cleaned))), + "missing_before": int(dataframe.isna().sum().sum()), + "missing_after": int(cleaned.isna().sum().sum()), + "duplicates_before": int(before_profile.get("duplicate_rows", 0)), + "duplicates_after": int(after_profile.get("duplicate_rows", 0)), + "before_health": before_health, + "after_health": after_health, + "health_delta": health_delta, + } diff --git a/src/framevitals/cli.py b/src/framevitals/cli.py index ec27df3..d7f3c04 100644 --- a/src/framevitals/cli.py +++ b/src/framevitals/cli.py @@ -3,6 +3,7 @@ from pathlib import Path from framevitals import __version__ +from framevitals.config import available_modules, available_presets def _add_output_argument(parser: argparse.ArgumentParser) -> None: @@ -10,7 +11,50 @@ def _add_output_argument(parser: argparse.ArgumentParser) -> None: "--output", type=Path, default=None, - help="Optional path to write the JSON result.", + help="Optional path to write the complete JSON result.", + ) + + +def _add_runtime_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--target", + default=None, + help="Optional target column. Overrides config values.", + ) + parser.add_argument( + "--mode", + choices=["quick", "standard", "deep", "research"], + default=None, + help="Analysis depth. Overrides preset/config values.", + ) + parser.add_argument( + "--preset", + choices=list(available_presets()), + default=None, + help="Built-in runtime preset.", + ) + parser.add_argument( + "--config", + type=Path, + default=None, + help="Optional FrameVitals TOML config path.", + ) + parser.add_argument( + "--workers", + type=int, + default=None, + help="Parallel worker count. Overrides config values.", + ) + parser.add_argument( + "--disable-module", + dest="disabled_modules", + action="append", + choices=list(available_modules()), + default=None, + help=( + "Disable an optional execution module. Repeat the flag to disable " + "multiple modules. Explicit flags override the config/preset list." + ), ) @@ -27,11 +71,18 @@ def _load_contract(path: Path) -> dict: return payload +def _parse_columns(value: str | None) -> list[str] | None: + if not value: + return None + columns = [item.strip() for item in value.split(",") if item.strip()] + return columns or None + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="framevitals", description=( - "Automated diagnostics, ML-readiness analysis, and drift " + "Automated diagnostics, ML-readiness analysis, validation, and drift " "comparison for tabular datasets." ), ) @@ -48,43 +99,136 @@ def build_parser() -> argparse.ArgumentParser: "analyze", help="Analyze a tabular dataset.", ) + analyze_parser.add_argument("file", type=Path, help="Path to the dataset.") + _add_runtime_arguments(analyze_parser) analyze_parser.add_argument( - "file", - type=Path, - help="Path to the dataset.", - ) - analyze_parser.add_argument( - "--target", + "--artifacts", + action=argparse.BooleanOptionalAction, default=None, - help="Optional target column.", + help="Enable/disable cleaned CSV and chart artifacts.", ) analyze_parser.add_argument( - "--mode", - choices=["quick", "standard", "deep", "research"], - default="standard", - help="Analysis depth.", + "--format", + choices=["terminal", "json"], + default="terminal", + help="Stdout format. JSON prints the complete result.", ) analyze_parser.add_argument( - "--artifacts", - action="store_true", - help="Persist cleaned CSV/chart artifacts.", + "--html-report", + type=Path, + default=None, + help="Optional path to write a self-contained HTML report.", ) _add_output_argument(analyze_parser) - compare_parser = subparsers.add_parser( - "compare", - help="Compare reference and current datasets for drift.", + inspect_parser = subparsers.add_parser( + "inspect", + help="Inspect dataset source metadata and execution capabilities.", ) - compare_parser.add_argument( - "reference", + inspect_parser.add_argument("file", type=Path, help="Path to the dataset.") + inspect_parser.add_argument( + "--format", + choices=["terminal", "json"], + default="terminal", + help="Source metadata output format.", + ) + _add_output_argument(inspect_parser) + + snapshot_parser = subparsers.add_parser( + "snapshot", + help="Analyze a dataset and emit compact monitoring state.", + ) + snapshot_parser.add_argument("file", type=Path, help="Path to the dataset.") + _add_runtime_arguments(snapshot_parser) + snapshot_parser.add_argument( + "--format", + choices=["terminal", "json"], + default="terminal", + help="Snapshot output format.", + ) + _add_output_argument(snapshot_parser) + + compare_snapshots_parser = subparsers.add_parser( + "compare-snapshots", + help="Compare two compact FrameVitals monitoring snapshots.", + ) + compare_snapshots_parser.add_argument( + "reference", type=Path, help="Reference/baseline snapshot JSON." + ) + compare_snapshots_parser.add_argument( + "current", type=Path, help="Current snapshot JSON." + ) + compare_snapshots_parser.add_argument( + "--format", + choices=["terminal", "json"], + default="terminal", + help="Snapshot-diff output format.", + ) + compare_snapshots_parser.add_argument( + "--fail-on-change", + action="store_true", + help="Return exit code 1 when the compact snapshot state changed.", + ) + _add_output_argument(compare_snapshots_parser) + + system_info_parser = subparsers.add_parser( + "system-info", + help="Inspect FrameVitals CPU/native/GPU execution capabilities.", + ) + system_info_parser.add_argument( + "--probe-gpu", + action=argparse.BooleanOptionalAction, + default=True, + help="Probe GPU/CUDA availability. Use --no-probe-gpu for support-safe output.", + ) + system_info_parser.add_argument( + "--format", + choices=["terminal", "json"], + default="terminal", + help="System-information output format.", + ) + _add_output_argument(system_info_parser) + + plan_parser = subparsers.add_parser( + "plan", + help="Preview applicable analyses without running heavy stages.", + ) + plan_parser.add_argument("file", type=Path, help="Path to the dataset.") + _add_runtime_arguments(plan_parser) + plan_parser.add_argument( + "--format", + choices=["terminal", "json"], + default="terminal", + help="Plan output format.", + ) + _add_output_argument(plan_parser) + + clean_parser = subparsers.add_parser( + "clean", + help="Inspect a conservative cleaning plan and optionally write cleaned CSV.", + ) + clean_parser.add_argument("file", type=Path, help="Path to the dataset.") + clean_parser.add_argument( + "--output", type=Path, - help="Reference/baseline dataset path.", + default=None, + help="Write the cleaned dataset to this CSV path. Omit for plan-only mode.", ) - compare_parser.add_argument( - "current", + clean_parser.add_argument( + "--plan-output", type=Path, - help="Current dataset path.", + default=None, + help="Optional path to write the inferred cleaning plan as JSON.", ) + + compare_parser = subparsers.add_parser( + "compare", + help="Compare reference and current datasets for schema/distribution drift.", + ) + compare_parser.add_argument( + "reference", type=Path, help="Reference/baseline dataset path." + ) + compare_parser.add_argument("current", type=Path, help="Current dataset path.") compare_parser.add_argument( "--columns", default=None, @@ -96,6 +240,18 @@ def build_parser() -> argparse.ArgumentParser: default=30, help="Maximum number of shared columns to compare.", ) + compare_parser.add_argument( + "--format", + choices=["json", "terminal"], + default="json", + help="Stdout format. JSON remains the default for backward compatibility.", + ) + compare_parser.add_argument( + "--fail-on", + choices=["minor", "moderate", "severe"], + default=None, + help="Return exit code 1 when drift reaches this severity. Disabled by default.", + ) _add_output_argument(compare_parser) infer_contract_parser = subparsers.add_parser( @@ -103,9 +259,42 @@ def build_parser() -> argparse.ArgumentParser: help="Infer a reusable data contract from a reference dataset.", ) infer_contract_parser.add_argument( - "file", - type=Path, - help="Path to the reference dataset.", + "file", type=Path, help="Path to the reference dataset." + ) + infer_contract_parser.add_argument( + "--numeric-tolerance", + type=float, + default=0.05, + help="Expand inferred numeric bounds by this fraction of the observed span.", + ) + infer_contract_parser.add_argument( + "--max-categories", + type=int, + default=20, + help="Infer allowed-value expectations up to this cardinality.", + ) + infer_contract_parser.add_argument( + "--null-fraction-tolerance", + type=float, + default=0.05, + help="Additional tolerated null fraction above the reference rate.", + ) + infer_contract_parser.add_argument( + "--infer-unique", + action=argparse.BooleanOptionalAction, + default=True, + help="Infer uniqueness constraints for sufficiently large fully unique columns.", + ) + infer_contract_parser.add_argument( + "--min-unique-rows", + type=int, + default=20, + help="Minimum non-null rows required before inferring uniqueness.", + ) + infer_contract_parser.add_argument( + "--allow-extra-columns", + action="store_true", + help="Allow columns not present in the reference contract.", ) _add_output_argument(infer_contract_parser) @@ -114,9 +303,7 @@ def build_parser() -> argparse.ArgumentParser: help="Validate a dataset against a JSON data contract.", ) validate_parser.add_argument( - "file", - type=Path, - help="Path to the dataset to validate.", + "file", type=Path, help="Path to the dataset to validate." ) validate_parser.add_argument( "--contract", @@ -124,17 +311,232 @@ def build_parser() -> argparse.ArgumentParser: required=True, help="Path to a JSON contract created by infer-contract.", ) + validate_parser.add_argument( + "--format", + choices=["json", "terminal"], + default="json", + help="Stdout format. JSON remains the default for backward compatibility.", + ) + validate_parser.add_argument( + "--fail-on-warn", + action="store_true", + help="Return exit code 1 for warning-only validation results.", + ) _add_output_argument(validate_parser) + gate_parser = subparsers.add_parser( + "gate", + help="Run contract and/or drift checks as one CI-friendly quality gate.", + ) + gate_parser.add_argument("current", type=Path, help="Current dataset path.") + gate_parser.add_argument( + "--reference", + type=Path, + default=None, + help="Optional reference dataset for drift checks.", + ) + gate_parser.add_argument( + "--contract", + type=Path, + default=None, + help="Optional JSON data contract for exact validation.", + ) + gate_parser.add_argument( + "--columns", + default=None, + help="Optional comma-separated columns to include in drift checks.", + ) + gate_parser.add_argument( + "--max-columns", + type=int, + default=30, + help="Maximum number of shared columns to compare for drift.", + ) + gate_parser.add_argument( + "--drift-warn-on", + choices=["stable", "minor", "moderate", "severe"], + default="moderate", + help="Drift severity that changes a passing gate to warning.", + ) + gate_parser.add_argument( + "--drift-fail-on", + choices=["stable", "minor", "moderate", "severe"], + default="severe", + help="Drift severity that fails the gate.", + ) + gate_parser.add_argument( + "--fail-on-validation-warning", + action="store_true", + help="Promote contract validation warnings to gate failures.", + ) + gate_parser.add_argument( + "--format", + choices=["json", "terminal"], + default="terminal", + help="Stdout format.", + ) + _add_output_argument(gate_parser) + + config_parser = subparsers.add_parser( + "config", + help="Resolve and inspect FrameVitals runtime configuration.", + ) + config_parser.add_argument( + "--file", + type=Path, + default=None, + help="Optional TOML config file.", + ) + config_parser.add_argument( + "--preset", + choices=list(available_presets()), + default=None, + help="Optional built-in preset to resolve before the config file.", + ) + return parser +def _write_json(payload: dict, output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + + def _emit(payload: dict, output: Path | None) -> None: - rendered = json.dumps(payload, indent=2, default=str) if output is not None: - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(rendered + "\n", encoding="utf-8") - print(rendered) + _write_json(payload, output) + print(json.dumps(payload, indent=2, default=str)) + + +def _render_source(source: dict) -> str: + rows = source.get("rows") + columns = source.get("columns") + size_bytes = source.get("size_bytes") + lines = [ + "FrameVitals source", + f"Name {source.get('name', 'unknown')}", + f"Kind {source.get('kind', 'unknown')}", + f"Format {source.get('format', 'unknown')}", + f"Rows {rows if rows is not None else 'unknown'}", + f"Columns {columns if columns is not None else 'unknown'}", + f"Size bytes {size_bytes if size_bytes is not None else 'unknown'}", + f"Materialized {'yes' if source.get('materialized') else 'no'}", + f"Projection {'yes' if source.get('supports_projection') else 'no'}", + f"Streaming {'yes' if source.get('supports_streaming') else 'no'}", + ] + return "\n".join(lines) + + +def _render_snapshot(snapshot: dict) -> str: + state = snapshot.get("state", {}) + source = snapshot.get("source", {}) + dataset = state.get("dataset", {}) + shape = dataset.get("shape", {}) + return "\n".join([ + "FrameVitals snapshot", + f"File {source.get('filename', 'unknown')}", + f"Mode {state.get('analysis_mode', 'unknown')}", + f"Rows {shape.get('rows', 'unknown')}", + f"Columns {shape.get('columns', 'unknown')}", + f"Fingerprint {snapshot.get('fingerprint', 'unknown')}", + ]) + + +def _render_snapshot_diff(report: dict) -> str: + schema = report.get("schema", {}) + findings = report.get("findings", {}) + return "\n".join([ + "FrameVitals snapshot diff", + f"Changed {'yes' if report.get('changed') else 'no'}", + f"Added columns {len(schema.get('added_columns', []))}", + f"Removed columns {len(schema.get('removed_columns', []))}", + f"Type changes {len(schema.get('type_changes', {}))}", + f"New findings {len(findings.get('new', []))}", + f"Resolved {len(findings.get('resolved', []))}", + ]) + + +def _render_system_info(info: dict) -> str: + native = info.get("native", {}) + gpu = info.get("gpu", {}) + lines = [ + "FrameVitals system info", + f"Python {info.get('python', 'unknown')}", + f"Backend {info.get('backend', 'unknown')}", + f"Native available: {native.get('available', False)}", + ] + if gpu: + lines.append(f"GPU available: {gpu.get('available', False)}") + return "\n".join(lines) + + +def _render_validation(report: dict) -> str: + summary = report.get("summary", {}) + lines = [ + "FrameVitals validation", + f"Status {str(report.get('status', 'unknown')).upper()}", + f"Columns checked {summary.get('columns_checked', 0)}", + f"Errors {summary.get('errors', 0)}", + f"Warnings {summary.get('warnings', 0)}", + ] + findings = report.get("findings", []) + if findings: + lines.append("") + lines.append("Top findings") + for finding in findings[:8]: + lines.append( + f"- [{str(finding.get('severity', 'error')).upper()}] " + f"{finding.get('column')}: {finding.get('message')}" + ) + return "\n".join(lines) + + +def _render_compare(report: dict) -> str: + if not report.get("available"): + return f"FrameVitals drift\nStatus UNAVAILABLE\nReason {report.get('reason')}" + + summary = report.get("summary", {}) + gate = report.get("gate", {}) + schema = report.get("schema", {}) + lines = [ + "FrameVitals drift", + f"Gate {str(gate.get('status', 'unknown')).upper()}", + f"Severity {str(summary.get('overall_verdict', 'unknown')).upper()}", + f"Columns checked {summary.get('n_columns_compared', 0)}", + f"Added columns {len(schema.get('added_columns', []))}", + f"Removed columns {len(schema.get('removed_columns', []))}", + f"Type changes {len(schema.get('dtype_changes', []))}", + ] + columns = report.get("columns", []) + notable = [ + entry + for entry in columns + if entry.get("drift_severity") in {"minor", "moderate", "severe"} + ] + if notable: + lines.append("") + lines.append("Top drift") + for entry in notable[:8]: + lines.append( + f"- [{str(entry.get('drift_severity')).upper()}] {entry.get('column')}" + ) + return "\n".join(lines) + + +def _render_gate(report: dict) -> str: + lines = [ + "FrameVitals gate", + f"Status {str(report.get('status', 'unknown')).upper()}", + f"Passed {'yes' if report.get('passed') else 'no'}", + f"Checks {', '.join(report.get('checks_run', [])) or 'none'}", + ] + reasons = report.get("reasons", []) + if reasons: + lines.append("") + lines.append("Reasons") + for reason in reasons[:10]: + lines.append(f"- {reason}") + return "\n".join(lines) def main() -> int: @@ -142,63 +544,247 @@ def main() -> int: args = parser.parse_args() if args.command == "analyze": - from framevitals.api import analyze + from framevitals.analysis_api import analyze report = analyze( args.file, target=args.target, mode=args.mode, artifacts=args.artifacts, + workers=args.workers, + preset=args.preset, + config=args.config, + disabled_modules=args.disabled_modules, + ) + + if args.output is not None: + report.to_json(args.output) + if args.html_report is not None: + report.to_html(args.html_report) + + if args.format == "json": + print(report.to_json()) + else: + print(report.summary_text()) + resolved = report.get("config", {}) + if resolved: + print( + "Config " + f"mode={resolved.get('mode')} " + f"workers={resolved.get('workers')} " + f"artifacts={resolved.get('artifacts')}" + ) + disabled = resolved.get("disabled_modules") or () + if disabled: + print(f"Disabled {', '.join(disabled)}") + if args.output is not None: + print(f"Full JSON {args.output}") + if args.html_report is not None: + print(f"HTML report {args.html_report}") + return 0 + + if args.command == "inspect": + from framevitals.sources import inspect_source + + result = inspect_source(args.file) + if args.output is not None: + _write_json(result, args.output) + if args.format == "json": + print(json.dumps(result, indent=2, default=str)) + else: + print(_render_source(result)) + if args.output is not None: + print(f"Source JSON {args.output}") + return 0 + + if args.command == "snapshot": + from framevitals.analysis_api import analyze + + report = analyze( + args.file, + target=args.target, + mode=args.mode, + artifacts=False, + workers=args.workers, + preset=args.preset, + config=args.config, + disabled_modules=args.disabled_modules, + ) + snapshot = report.snapshot(args.output) + if args.format == "json": + print(snapshot.to_json()) + else: + print(_render_snapshot(snapshot)) + if args.output is not None: + print(f"Snapshot JSON {args.output}") + return 0 + + if args.command == "compare-snapshots": + from framevitals.snapshots import compare_snapshots, load_snapshot + + reference = load_snapshot(args.reference) + current = load_snapshot(args.current) + result = compare_snapshots(reference, current) + if args.output is not None: + _write_json(result, args.output) + if args.format == "json": + print(json.dumps(result, indent=2, default=str)) + else: + print(_render_snapshot_diff(result)) + if args.fail_on_change and result.get("changed"): + return 1 + return 0 + + if args.command == "system-info": + from framevitals.acceleration import system_info + + result = system_info(probe_gpu=args.probe_gpu) + if args.output is not None: + _write_json(result, args.output) + if args.format == "json": + print(json.dumps(result, indent=2, default=str)) + else: + print(_render_system_info(result)) + if args.output is not None: + print(f"System JSON {args.output}") + return 0 + + if args.command == "plan": + from framevitals.planning_api import plan + + result = plan( + args.file, + target=args.target, + mode=args.mode, + workers=args.workers, + preset=args.preset, + config=args.config, + disabled_modules=args.disabled_modules, ) + if args.output is not None: + _write_json(dict(result), args.output) + if args.format == "json": + print(json.dumps(dict(result), indent=2, default=str)) + else: + print(result.explain_text()) + disabled = result.get("config", {}).get("disabled_modules") or () + if disabled: + print(f"Disabled modules: {', '.join(disabled)}") + if args.output is not None: + print(f"Plan JSON {args.output}") + return 0 + + if args.command == "clean": + from framevitals.operations import clean, plan_cleaning + from framevitals.security import sanitize_csv_value - summary = { - "file": report.get("filename"), - "mode": report.get("analysis_mode"), - "health": report.get("health"), - "ml_readiness": report.get("ml_readiness"), - "dataset_signals": report.get("dataset_signals"), - "artifacts_enabled": report.get("artifacts_enabled"), - "timings_ms": report.get("timings_ms"), + cleaning_plan = plan_cleaning(args.file) + if args.plan_output is not None: + cleaning_plan.to_json(args.plan_output) + + payload = { + "plan": dict(cleaning_plan), + "summary": cleaning_plan.summary(), + "cleaned_output": str(args.output) if args.output is not None else None, } - _emit(summary, args.output) + + if args.output is not None: + if args.output.suffix.lower() != ".csv": + raise ValueError("The clean command currently writes CSV output only.") + cleaned = clean(args.file, plan=cleaning_plan) + args.output.parent.mkdir(parents=True, exist_ok=True) + cleaned.map(sanitize_csv_value).to_csv(args.output, index=False) + + print(json.dumps(payload, indent=2, default=str)) return 0 if args.command == "compare": - from framevitals.api import compare - - columns = None - if args.columns: - columns = [ - value.strip() - for value in args.columns.split(",") - if value.strip() - ] + from framevitals.drift_analysis import severity_at_least + from framevitals.operations import compare report = compare( args.reference, args.current, - columns=columns, + columns=_parse_columns(args.columns), max_columns=args.max_columns, ) - _emit(report, args.output) + if args.output is not None: + _write_json(report, args.output) + if args.format == "terminal": + print(_render_compare(report)) + else: + print(json.dumps(report, indent=2, default=str)) + + if args.fail_on and severity_at_least( + report.get("gate", {}).get("severity", "unknown"), + args.fail_on, + ): + return 1 return 0 if args.command == "infer-contract": - from framevitals.api import infer_contract + from framevitals.operations import infer_contract - report = infer_contract(args.file) + report = infer_contract( + args.file, + numeric_tolerance=args.numeric_tolerance, + max_categories=args.max_categories, + null_fraction_tolerance=args.null_fraction_tolerance, + infer_unique=args.infer_unique, + min_unique_rows=args.min_unique_rows, + allow_extra_columns=args.allow_extra_columns, + ) _emit(report, args.output) return 0 if args.command == "validate": - from framevitals.api import validate + from framevitals.operations import validate report = validate( args.file, _load_contract(args.contract), ) - _emit(report, args.output) - return 0 if report["valid"] else 1 + if args.output is not None: + _write_json(report, args.output) + if args.format == "terminal": + print(_render_validation(report)) + else: + print(json.dumps(report, indent=2, default=str)) + + if report.get("status") == "fail": + return 2 + if report.get("status") == "warn" and args.fail_on_warn: + return 1 + return 0 + + if args.command == "gate": + from framevitals.operations import gate + + contract = _load_contract(args.contract) if args.contract is not None else None + report = gate( + args.current, + reference=args.reference, + contract=contract, + columns=_parse_columns(args.columns), + max_columns=args.max_columns, + drift_warn_on=args.drift_warn_on, + drift_fail_on=args.drift_fail_on, + fail_on_validation_warning=args.fail_on_validation_warning, + ) + if args.output is not None: + _write_json(report, args.output) + if args.format == "json": + print(json.dumps(report, indent=2, default=str)) + else: + print(_render_gate(report)) + return 0 if report.get("passed") else 1 + + if args.command == "config": + from framevitals.config import resolve_config + + resolved = resolve_config(args.file, preset=args.preset) + print(json.dumps(resolved.to_dict(), indent=2)) + return 0 parser.print_help() return 0 diff --git a/src/framevitals/cli_entry.py b/src/framevitals/cli_entry.py new file mode 100644 index 0000000..ec6c099 --- /dev/null +++ b/src/framevitals/cli_entry.py @@ -0,0 +1,14 @@ +"""Backward-compatible CLI entrypoint. + +The installed console script now points directly to :mod:`framevitals.cli`. +This module remains as a lightweight compatibility alias for callers that +imported ``framevitals.cli_entry.main`` during the 0.x series. +""" + +from __future__ import annotations + + +def main() -> int: + from framevitals.cli import main as cli_main + + return cli_main() diff --git a/src/framevitals/column_roles.py b/src/framevitals/column_roles.py index 21801b5..7c06f40 100644 --- a/src/framevitals/column_roles.py +++ b/src/framevitals/column_roles.py @@ -1,17 +1,19 @@ """ Column Role Inference Engine ============================ -Assigns semantic roles to each column based on name keywords, -dtype, unique ratio, missingness, and statistical properties. - -Each column receives a SET of roles (not a single label), -enabling downstream modules to make informed decisions. +Assigns semantic roles to each column based on name keywords, dtype, unique +ratio, missingness, statistical properties, and bounded value-pattern samples. """ +from __future__ import annotations + import re +import numpy as np import pandas as pd +from framevitals.semantic_types import infer_semantic_types + ID_KEYWORDS = [ "id", "uuid", "hash", "key", "identifier", "index", "roll", "roll_number", "rollno", "roll number", @@ -56,14 +58,56 @@ "pass", "fail", "approved", "rejected", ] +SEMANTIC_ROLE_MAP = { + "email": "email_like", + "url": "url_like", + "uuid": "uuid_like", + "ip_address": "ip_address_like", + "phone": "phone_like", + "percentage": "percentage_like", + "currency": "currency_like", + "json": "json_like", + "boolean_token": "boolean_token_like", +} -def _name_matches(column_name: str, keywords: list) -> bool: - lower = column_name.lower().replace("-", "_") - return any(kw in lower for kw in keywords) +def _normalise_name(value: object) -> tuple[str, tuple[str, ...]]: + normalized = re.sub(r"[^a-z0-9]+", "_", str(value).strip().lower()).strip("_") + return normalized, tuple(token for token in normalized.split("_") if token) -def _safe_unique_ratio(series: pd.Series) -> float: - return series.nunique(dropna=True) / max(len(series), 1) + +def _name_matches(column_name: str, keywords: list) -> bool: + """Match keyword tokens/phrases without unsafe substring collisions. + + This prevents names such as ``paid_amount`` from matching ``id`` and + ``average_score`` from matching ``age`` while retaining conventional names + such as ``customer_id``, ``created_at``, and ``roll_number``. + """ + normalized, tokens = _normalise_name(column_name) + token_set = set(tokens) + + for keyword in keywords: + keyword_normalized, keyword_tokens = _normalise_name(keyword) + if not keyword_normalized: + continue + if normalized == keyword_normalized: + return True + if len(keyword_tokens) == 1 and keyword_tokens[0] in token_set: + return True + if len(keyword_tokens) > 1: + width = len(keyword_tokens) + for start in range(0, len(tokens) - width + 1): + if tokens[start : start + width] == keyword_tokens: + return True + return False + + +def _bounded_text_sample(series: pd.Series, max_rows: int = 500) -> pd.Series: + clean = series.dropna() + if len(clean) <= max_rows: + return clean.astype(str) + positions = np.linspace(0, len(clean) - 1, num=max_rows, dtype=np.int64) + return clean.iloc[np.unique(positions)].astype(str) def _classify_missingness(missing_percent: float) -> str: @@ -96,6 +140,13 @@ def _infer_single_column_roles(column: str, series: pd.Series, rows: int) -> dic or isinstance(series.dtype, pd.CategoricalDtype) ) + semantic = ( + infer_semantic_types(series) + if is_text + else {"primary": None, "candidates": [], "sample_size": 0} + ) + semantic_primary = semantic.get("primary") + if is_numeric: roles.add("numeric") if is_bool: @@ -131,16 +182,26 @@ def _infer_single_column_roles(column: str, series: pd.Series, rows: int) -> dic if _name_matches(column, TARGET_HINT_KEYWORDS): roles.add("target_hint") + if semantic_primary in SEMANTIC_ROLE_MAP: + roles.add(SEMANTIC_ROLE_MAP[semantic_primary]) + if semantic_primary in {"email", "phone", "ip_address"}: + roles.add("sensitive") + if semantic_primary == "uuid": + roles.add("id_like") + if semantic_primary == "currency": + roles.add("price_like") + if not is_numeric and not is_bool: - sample = series.dropna().astype(str).head(30) + sample = _bounded_text_sample(series, 30) if len(sample) > 0: parsed = pd.to_datetime(sample, errors="coerce", format="mixed") if parsed.notna().mean() >= 0.7: roles.add("time_like") if is_text: - lengths = series.dropna().astype(str).str.len() - if len(lengths) > 0: + sample = _bounded_text_sample(series, 500) + if len(sample) > 0: + lengths = sample.str.len() avg_len = float(lengths.mean()) max_len = int(lengths.max()) if avg_len > 50 or max_len > 200: @@ -169,6 +230,9 @@ def _infer_single_column_roles(column: str, series: pd.Series, rows: int) -> dic "non_missing_count": non_missing, "is_numeric": bool(is_numeric), "is_categorical": bool(is_text or is_bool), + "semantic_type": semantic_primary, + "semantic_candidates": semantic.get("candidates", []), + "semantic_sample_size": int(semantic.get("sample_size", 0)), } @@ -234,6 +298,15 @@ def summarize_roles(column_roles: dict) -> dict: column_roles, "regression_target_candidate" ), "sensitive": get_columns_with_role(column_roles, "sensitive"), + "email_like": get_columns_with_role(column_roles, "email_like"), + "url_like": get_columns_with_role(column_roles, "url_like"), + "uuid_like": get_columns_with_role(column_roles, "uuid_like"), + "ip_address_like": get_columns_with_role(column_roles, "ip_address_like"), + "phone_like": get_columns_with_role(column_roles, "phone_like"), + "percentage_like": get_columns_with_role(column_roles, "percentage_like"), + "currency_like": get_columns_with_role(column_roles, "currency_like"), + "json_like": get_columns_with_role(column_roles, "json_like"), + "boolean_token_like": get_columns_with_role(column_roles, "boolean_token_like"), "high_missing": [ col for col, info in column_roles.items() if info["missing_percent"] >= 20 diff --git a/src/framevitals/config.py b/src/framevitals/config.py new file mode 100644 index 0000000..1f4c10a --- /dev/null +++ b/src/framevitals/config.py @@ -0,0 +1,267 @@ +"""Runtime configuration for FrameVitals analysis. + +The configuration layer controls both analysis depth/resources and optional +pipeline modules. Defaults preserve historical behaviour; users can explicitly +disable expensive or irrelevant modules without changing the stable result +shape or maintaining a second configuration system. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Any, Mapping +import tomllib + + +VALID_MODES = {"quick", "standard", "deep", "research"} +VALID_MODULES = { + "quality_diagnostics", + "deep_statistics", + "anomaly_detection", + "time_series", + "text_profile", + "target_intelligence", + "modeling", + "explainability", + "cleaning", + "charts", + "ai", +} + +PRESETS: dict[str, dict[str, Any]] = { + "quick": {"mode": "quick", "workers": 2, "artifacts": False}, + "standard": {"mode": "standard", "workers": 4, "artifacts": False}, + "deep": {"mode": "deep", "workers": 4, "artifacts": False}, + "research": {"mode": "research", "workers": 4, "artifacts": False}, + "ci": { + "mode": "standard", + "workers": 2, + "artifacts": False, + "disabled_modules": ("modeling", "explainability", "charts", "ai"), + }, +} + + +@dataclass(frozen=True, slots=True) +class AnalysisConfig: + """Resolved configuration consumed by the analysis pipeline.""" + + mode: str = "standard" + target: str | None = None + artifacts: bool = False + workers: int = 4 + disabled_modules: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.mode not in VALID_MODES: + raise ValueError( + f"Invalid analysis mode '{self.mode}'. " + f"Choose from: {', '.join(sorted(VALID_MODES))}" + ) + if self.workers < 1: + raise ValueError("workers must be at least 1.") + + modules = tuple(dict.fromkeys(self.disabled_modules)) + unknown = sorted(set(modules) - VALID_MODULES) + if unknown: + raise ValueError( + "Unknown FrameVitals module(s): " + f"{', '.join(unknown)}. Choose from: {', '.join(sorted(VALID_MODULES))}" + ) + object.__setattr__(self, "disabled_modules", modules) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + def module_enabled(self, name: str) -> bool: + if name not in VALID_MODULES: + raise ValueError(f"Unknown FrameVitals module: {name}") + return name not in self.disabled_modules + + +ConfigInput = AnalysisConfig | Mapping[str, Any] | str | Path | None + + +def available_presets() -> tuple[str, ...]: + """Return built-in preset names in deterministic order.""" + return tuple(PRESETS) + + +def available_modules() -> tuple[str, ...]: + """Return configurable execution module names in deterministic order.""" + return tuple(sorted(VALID_MODULES)) + + +def _read_toml(path: str | Path) -> dict[str, Any]: + source = Path(path) + if not source.exists(): + raise FileNotFoundError(f"FrameVitals config not found: {source}") + if not source.is_file(): + raise ValueError(f"Expected a config file, got: {source}") + + try: + with source.open("rb") as handle: + payload = tomllib.load(handle) + except tomllib.TOMLDecodeError as exc: + raise ValueError(f"Invalid TOML in FrameVitals config: {source}") from exc + + if not isinstance(payload, dict): + raise ValueError("FrameVitals config must contain a TOML table.") + return payload + + +def _coerce_disabled_modules(value: Any) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, str) or not isinstance(value, (list, tuple, set)): + raise ValueError("disabled_modules must be a list/tuple of module names.") + return tuple(str(item) for item in value) + + +def _extract_values( + mapping: Mapping[str, Any], +) -> tuple[str | None, dict[str, Any], dict[str, bool]]: + """Extract scalar values and module overrides from config mappings.""" + analysis = mapping.get("analysis", {}) + resources = mapping.get("resources", {}) + modules = mapping.get("modules", {}) + + if not isinstance(analysis, Mapping): + raise ValueError("[analysis] must be a TOML table/object.") + if not isinstance(resources, Mapping): + raise ValueError("[resources] must be a TOML table/object.") + if not isinstance(modules, Mapping): + raise ValueError("[modules] must be a TOML table/object.") + + preset = analysis.get("preset", mapping.get("preset")) + values: dict[str, Any] = {} + + for key in ("mode", "target", "artifacts"): + if key in analysis: + values[key] = analysis[key] + elif key in mapping: + values[key] = mapping[key] + + if "workers" in resources: + values["workers"] = resources["workers"] + elif "workers" in mapping: + values["workers"] = mapping["workers"] + + if "disabled_modules" in analysis: + values["disabled_modules"] = _coerce_disabled_modules( + analysis["disabled_modules"] + ) + elif "disabled_modules" in mapping: + values["disabled_modules"] = _coerce_disabled_modules( + mapping["disabled_modules"] + ) + + module_overrides: dict[str, bool] = {} + for name, enabled in modules.items(): + if name not in VALID_MODULES: + raise ValueError(f"Unknown FrameVitals module in [modules]: {name}") + if not isinstance(enabled, bool): + raise ValueError(f"[modules].{name} must be true or false.") + module_overrides[name] = enabled + + return str(preset) if preset is not None else None, values, module_overrides + + +def _preset_values(name: str | None) -> dict[str, Any]: + if name is None: + return {} + if name not in PRESETS: + raise ValueError( + f"Unknown FrameVitals preset '{name}'. " + f"Choose from: {', '.join(available_presets())}" + ) + return dict(PRESETS[name]) + + +def _apply_module_overrides( + values: dict[str, Any], + overrides: Mapping[str, bool], +) -> None: + disabled = list(_coerce_disabled_modules(values.get("disabled_modules", ()))) + for name, enabled in overrides.items(): + if enabled: + disabled = [item for item in disabled if item != name] + elif name not in disabled: + disabled.append(name) + values["disabled_modules"] = tuple(disabled) + + +def resolve_config( + config: ConfigInput = None, + *, + preset: str | None = None, + mode: str | None = None, + target: str | None = None, + artifacts: bool | None = None, + workers: int | None = None, + disabled_modules: list[str] | tuple[str, ...] | None = None, +) -> AnalysisConfig: + """Resolve defaults, preset, config file/object, then explicit overrides. + + Precedence, from lowest to highest, is: + + 1. FrameVitals defaults + 2. explicit ``preset=`` argument + 3. configuration file/mapping/object (including ``[modules]`` booleans) + 4. explicit function/CLI arguments + """ + values: dict[str, Any] = AnalysisConfig().to_dict() + values.update(_preset_values(preset)) + + if isinstance(config, AnalysisConfig): + values.update(config.to_dict()) + elif isinstance(config, (str, Path)): + config_preset, config_values, module_overrides = _extract_values( + _read_toml(config) + ) + if config_preset is not None: + values.update(_preset_values(config_preset)) + values.update(config_values) + _apply_module_overrides(values, module_overrides) + elif isinstance(config, Mapping): + config_preset, config_values, module_overrides = _extract_values(config) + if config_preset is not None: + values.update(_preset_values(config_preset)) + values.update(config_values) + _apply_module_overrides(values, module_overrides) + elif config is not None: + raise TypeError( + "config must be an AnalysisConfig, mapping, TOML path, or None." + ) + + explicit = { + "mode": mode, + "target": target, + "artifacts": artifacts, + "workers": workers, + "disabled_modules": ( + tuple(disabled_modules) if disabled_modules is not None else None + ), + } + values.update({key: value for key, value in explicit.items() if value is not None}) + + try: + values["workers"] = int(values["workers"]) + except (TypeError, ValueError) as exc: + raise ValueError("workers must be an integer.") from exc + + if not isinstance(values["artifacts"], bool): + raise ValueError("artifacts must be true or false.") + if values["target"] is not None and not isinstance(values["target"], str): + raise ValueError("target must be a column name string or null.") + + values["disabled_modules"] = _coerce_disabled_modules( + values.get("disabled_modules", ()) + ) + return AnalysisConfig(**values) + + +def with_overrides(config: AnalysisConfig, **changes: Any) -> AnalysisConfig: + """Return a validated copy of an existing resolved configuration.""" + return replace(config, **changes) diff --git a/src/framevitals/contracts.py b/src/framevitals/contracts.py index 3bf617b..3f04264 100644 --- a/src/framevitals/contracts.py +++ b/src/framevitals/contracts.py @@ -1,7 +1,15 @@ -"""Inference and validation helpers for lightweight data contracts.""" +"""Inference and validation helpers for reusable data contracts. + +Contracts are intentionally lightweight JSON-compatible dictionaries. Version 2 +adds tolerant numeric bounds, optional columns, low-cardinality value sets, +null-rate ceilings, uniqueness hints, and normalized validation severities while +remaining able to validate version-1 contracts created by earlier FrameVitals +builds. +""" from __future__ import annotations +from collections import Counter from collections.abc import Mapping from typing import Any @@ -9,7 +17,8 @@ import pandas as pd from pandas.api import types as pdt -CONTRACT_VERSION = 1 +CONTRACT_VERSION = 2 +_SUPPORTED_CONTRACT_VERSIONS = {1, 2} _TYPE_FAMILIES = { "boolean", "datetime", @@ -17,6 +26,7 @@ "number", "string", } +_SEVERITIES = {"ignore", "warning", "error"} def _require_contractible_columns(dataframe: pd.DataFrame) -> None: @@ -44,48 +54,150 @@ def _type_family(series: pd.Series) -> str: return "string" -def _numeric_bounds(series: pd.Series) -> dict[str, int | float]: +def _json_scalar(value: Any) -> Any: + if isinstance(value, np.generic): + return value.item() + if isinstance(value, pd.Timestamp): + return value.isoformat() + return value + + +def _numeric_bounds( + series: pd.Series, + *, + tolerance: float, + family: str, +) -> dict[str, int | float]: values = pd.to_numeric(series, errors="coerce").dropna() values = values[np.isfinite(values)] if values.empty: return {} - minimum = values.min() - maximum = values.max() + observed_min = float(values.min()) + observed_max = float(values.max()) + span = observed_max - observed_min + if span <= 0: + span = max(abs(observed_min), 1.0) + margin = span * tolerance + + minimum = observed_min - margin + maximum = observed_max + margin + if family == "integer": + minimum = int(np.floor(minimum)) + maximum = int(np.ceil(maximum)) + return { - "minimum": minimum.item() if hasattr(minimum, "item") else minimum, - "maximum": maximum.item() if hasattr(maximum, "item") else maximum, + "minimum": _json_scalar(minimum), + "maximum": _json_scalar(maximum), + "observed_minimum": _json_scalar(observed_min), + "observed_maximum": _json_scalar(observed_max), } -def infer_contract(dataframe: pd.DataFrame) -> dict[str, Any]: +def _allowed_values(series: pd.Series, *, max_categories: int) -> list[Any] | None: + clean = series.dropna() + if clean.empty: + return None + unique = clean.unique() + if len(unique) > max_categories: + return None + values = [_json_scalar(value) for value in unique.tolist()] + return sorted(values, key=lambda value: str(value)) + + +def infer_contract( + dataframe: pd.DataFrame, + *, + numeric_tolerance: float = 0.05, + max_categories: int = 20, + null_fraction_tolerance: float = 0.05, + infer_unique: bool = True, + min_unique_rows: int = 20, + allow_extra_columns: bool = False, +) -> dict[str, Any]: """Create a JSON-serializable contract from a reference dataset. - The inferred contract captures schema and basic quality constraints that - are stable enough to use as an automated data gate: required columns, - broad type families, nullability, and finite numeric bounds. + Inference is deliberately conservative. Numeric extrema are expanded by a + configurable tolerance so ordinary future observations do not fail merely + because they exceed a reference sample's exact minimum or maximum. Small + categorical domains are recorded as warning-level expectations. Fully + unique columns are marked unique only when enough rows exist to make that + inference meaningful. """ _require_contractible_columns(dataframe) + if numeric_tolerance < 0: + raise ValueError("numeric_tolerance must be non-negative.") + if max_categories < 1: + raise ValueError("max_categories must be at least 1.") + if not 0 <= null_fraction_tolerance <= 1: + raise ValueError("null_fraction_tolerance must be between 0 and 1.") + if min_unique_rows < 2: + raise ValueError("min_unique_rows must be at least 2.") + columns: dict[str, dict[str, Any]] = {} for name in dataframe.columns: series = dataframe[name] family = _type_family(series) + null_fraction = float(series.isna().mean()) if len(series) else 0.0 + clean = series.dropna() + specification: dict[str, Any] = { "type": family, + "required": True, "nullable": bool(series.isna().any()), + "max_null_fraction": round( + min(1.0, null_fraction + null_fraction_tolerance), + 6, + ), } + if family in {"integer", "number"}: - specification.update(_numeric_bounds(series)) + specification.update( + _numeric_bounds( + series, + tolerance=numeric_tolerance, + family=family, + ) + ) + elif family in {"string", "boolean"}: + allowed = _allowed_values(series, max_categories=max_categories) + if allowed is not None: + specification["allowed_values"] = allowed + specification["allowed_values_severity"] = "warning" + + if ( + infer_unique + and len(clean) >= min_unique_rows + and int(clean.nunique(dropna=True)) == len(clean) + ): + specification["unique"] = True + columns[name] = specification return { "version": CONTRACT_VERSION, - "allow_extra_columns": False, + "allow_extra_columns": bool(allow_extra_columns), + "extra_columns_severity": "ignore" if allow_extra_columns else "error", + "inference": { + "numeric_tolerance": float(numeric_tolerance), + "max_categories": int(max_categories), + "null_fraction_tolerance": float(null_fraction_tolerance), + "infer_unique": bool(infer_unique), + "min_unique_rows": int(min_unique_rows), + }, "columns": columns, } +def _normalise_severity(value: Any, *, field: str) -> str: + severity = str(value).strip().lower() + if severity not in _SEVERITIES: + choices = ", ".join(sorted(_SEVERITIES)) + raise ValueError(f"contract field '{field}' must be one of: {choices}.") + return severity + + def _normalise_contract(contract: Mapping[str, Any]) -> dict[str, Any]: if not isinstance(contract, Mapping): raise TypeError("contract must be a mapping produced by infer_contract().") @@ -94,14 +206,20 @@ def _normalise_contract(contract: Mapping[str, Any]) -> dict[str, Any]: if not isinstance(columns, Mapping) or not columns: raise ValueError("contract must contain a non-empty 'columns' mapping.") - version = contract.get("version", CONTRACT_VERSION) - if version != CONTRACT_VERSION: + version = contract.get("version", 1) + if version not in _SUPPORTED_CONTRACT_VERSIONS: raise ValueError(f"Unsupported contract version: {version!r}.") allow_extra_columns = contract.get("allow_extra_columns", False) if not isinstance(allow_extra_columns, bool): raise ValueError("contract field 'allow_extra_columns' must be a boolean.") + default_extra_severity = "ignore" if allow_extra_columns else "error" + extra_columns_severity = _normalise_severity( + contract.get("extra_columns_severity", default_extra_severity), + field="extra_columns_severity", + ) + normalised_columns: dict[str, dict[str, Any]] = {} for name, specification in columns.items(): if not isinstance(name, str): @@ -114,14 +232,34 @@ def _normalise_contract(contract: Mapping[str, Any]) -> dict[str, Any]: choices = ", ".join(sorted(_TYPE_FAMILIES)) raise ValueError(f"contract type for '{name}' must be one of: {choices}.") + required = specification.get("required", True) nullable = specification.get("nullable", True) - if not isinstance(nullable, bool): - raise ValueError(f"contract field 'nullable' for '{name}' must be a boolean.") + unique = specification.get("unique", False) + for field_name, field_value in { + "required": required, + "nullable": nullable, + "unique": unique, + }.items(): + if not isinstance(field_value, bool): + raise ValueError( + f"contract field '{field_name}' for '{name}' must be a boolean." + ) - normalised = { + normalised: dict[str, Any] = { "type": family, + "required": required, "nullable": nullable, + "unique": unique, } + + max_null_fraction = specification.get("max_null_fraction") + if max_null_fraction is not None: + if not isinstance(max_null_fraction, int | float) or not 0 <= max_null_fraction <= 1: + raise ValueError( + f"contract field 'max_null_fraction' for '{name}' must be between 0 and 1." + ) + normalised["max_null_fraction"] = float(max_null_fraction) + for bound in ("minimum", "maximum"): value = specification.get(bound) if value is not None: @@ -135,11 +273,29 @@ def _normalise_contract(contract: Mapping[str, Any]) -> dict[str, Any]: upper = normalised.get("maximum") if lower is not None and upper is not None and lower > upper: raise ValueError(f"contract minimum exceeds maximum for '{name}'.") + + allowed_values = specification.get("allowed_values") + if allowed_values is not None: + if family not in {"string", "boolean", "integer", "number"}: + raise ValueError( + f"'allowed_values' is not supported for column '{name}' of type '{family}'." + ) + if not isinstance(allowed_values, list) or not allowed_values: + raise ValueError( + f"contract field 'allowed_values' for '{name}' must be a non-empty list." + ) + normalised["allowed_values"] = allowed_values + normalised["allowed_values_severity"] = _normalise_severity( + specification.get("allowed_values_severity", "error"), + field=f"allowed_values_severity.{name}", + ) + normalised_columns[name] = normalised return { "version": version, "allow_extra_columns": allow_extra_columns, + "extra_columns_severity": extra_columns_severity, "columns": normalised_columns, } @@ -150,18 +306,46 @@ def _compatible_type(expected: str, actual: str) -> bool: return expected == actual -def _finding(code: str, column: str, message: str, **details: Any) -> dict[str, Any]: +def _finding( + code: str, + column: str, + message: str, + *, + severity: str = "error", + **details: Any, +) -> dict[str, Any]: finding: dict[str, Any] = { "code": code, "column": column, + "severity": severity, "message": message, } finding.update(details) return finding +def _append_finding( + errors: list[dict[str, Any]], + warnings: list[dict[str, Any]], + finding: dict[str, Any], +) -> None: + severity = finding.get("severity", "error") + if severity == "ignore": + return + if severity == "warning": + warnings.append(finding) + else: + errors.append(finding) + + def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> dict[str, Any]: - """Validate a dataset against an inferred or explicit data contract.""" + """Validate a dataset against an inferred or explicit data contract. + + Validation is aggregated rather than fail-fast: every applicable violation + is collected so CI, notebooks, and users can fix a dataset in one pass. + ``valid`` remains ``True`` when only warnings are present; ``status`` + distinguishes ``pass``, ``warn``, and ``fail``. + """ _require_contractible_columns(dataframe) normalised = _normalise_contract(contract) specifications = normalised["columns"] @@ -171,24 +355,29 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d actual_columns = set(dataframe.columns) expected_columns = set(specifications) for name in sorted(expected_columns - actual_columns): - errors.append( - _finding( - "missing_column", - name, - f"Required column '{name}' is missing.", - ) - ) - - if not normalised["allow_extra_columns"]: - for name in sorted(actual_columns - expected_columns): + specification = specifications[name] + if specification.get("required", True): errors.append( _finding( - "unexpected_column", + "missing_column", name, - f"Column '{name}' is not defined by the contract.", + f"Required column '{name}' is missing.", ) ) + extra_severity = normalised["extra_columns_severity"] + for name in sorted(actual_columns - expected_columns): + _append_finding( + errors, + warnings, + _finding( + "unexpected_column", + name, + f"Column '{name}' is not defined by the contract.", + severity=extra_severity, + ), + ) + for name in sorted(expected_columns & actual_columns): specification = specifications[name] series = dataframe[name] @@ -208,6 +397,7 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d continue null_count = int(series.isna().sum()) + null_fraction = float(series.isna().mean()) if len(series) else 0.0 if not specification["nullable"] and null_count: errors.append( _finding( @@ -215,8 +405,62 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d name, f"Column '{name}' does not allow null values ({null_count} found).", null_count=null_count, + null_fraction=round(null_fraction, 6), ) ) + else: + max_null_fraction = specification.get("max_null_fraction") + if max_null_fraction is not None and null_fraction > max_null_fraction: + warnings.append( + _finding( + "null_fraction_violation", + name, + ( + f"Column '{name}' has {null_fraction:.1%} null values, above " + f"its expected ceiling of {max_null_fraction:.1%}." + ), + severity="warning", + null_fraction=round(null_fraction, 6), + max_null_fraction=max_null_fraction, + ) + ) + + if specification.get("unique"): + clean = series.dropna() + duplicate_count = int(clean.duplicated().sum()) + if duplicate_count: + errors.append( + _finding( + "uniqueness_violation", + name, + f"Column '{name}' must be unique ({duplicate_count} duplicate values found).", + duplicate_count=duplicate_count, + ) + ) + + allowed_values = specification.get("allowed_values") + if allowed_values is not None: + allowed = set(allowed_values) + observed = series.dropna().unique().tolist() + unexpected_values = [ + _json_scalar(value) + for value in observed + if _json_scalar(value) not in allowed + ] + if unexpected_values: + severity = specification.get("allowed_values_severity", "error") + _append_finding( + errors, + warnings, + _finding( + "allowed_values_violation", + name, + f"Column '{name}' contains values outside its expected domain.", + severity=severity, + unexpected_values=unexpected_values[:20], + unexpected_count=len(unexpected_values), + ), + ) if expected_type not in {"integer", "number"}: continue @@ -226,45 +470,54 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d if values.empty: continue - observed_minimum = values.min() - observed_maximum = values.max() - observed_minimum = ( - observed_minimum.item() - if hasattr(observed_minimum, "item") - else observed_minimum - ) - observed_maximum = ( - observed_maximum.item() - if hasattr(observed_maximum, "item") - else observed_maximum - ) + observed_minimum = _json_scalar(values.min()) + observed_maximum = _json_scalar(values.max()) minimum = specification.get("minimum") - if minimum is not None and observed_minimum < minimum: - errors.append( - _finding( - "minimum_violation", - name, - f"Column '{name}' contains values below its minimum of {minimum}.", - minimum=minimum, - observed_minimum=observed_minimum, + if minimum is not None: + below_count = int((values < minimum).sum()) + if below_count: + errors.append( + _finding( + "minimum_violation", + name, + f"Column '{name}' contains {below_count} values below its minimum of {minimum}.", + minimum=minimum, + observed_minimum=observed_minimum, + violation_count=below_count, + ) ) - ) maximum = specification.get("maximum") - if maximum is not None and observed_maximum > maximum: - errors.append( - _finding( - "maximum_violation", - name, - f"Column '{name}' contains values above its maximum of {maximum}.", - maximum=maximum, - observed_maximum=observed_maximum, + if maximum is not None: + above_count = int((values > maximum).sum()) + if above_count: + errors.append( + _finding( + "maximum_violation", + name, + f"Column '{name}' contains {above_count} values above its maximum of {maximum}.", + maximum=maximum, + observed_maximum=observed_maximum, + violation_count=above_count, + ) ) - ) + + all_findings = errors + warnings + code_counts = dict(Counter(item["code"] for item in all_findings)) + failed_columns = sorted({item["column"] for item in errors}) + warning_columns = sorted({item["column"] for item in warnings}) + + if errors: + status = "fail" + elif warnings: + status = "warn" + else: + status = "pass" return { "valid": not errors, + "status": status, "contract_version": normalised["version"], "data": { "rows": int(len(dataframe)), @@ -274,7 +527,12 @@ def validate_contract(dataframe: pd.DataFrame, contract: Mapping[str, Any]) -> d "columns_checked": len(specifications), "errors": len(errors), "warnings": len(warnings), + "findings": len(all_findings), + "failed_columns": failed_columns, + "warning_columns": warning_columns, + "code_counts": code_counts, }, "errors": errors, "warnings": warnings, + "findings": all_findings, } diff --git a/src/framevitals/dataset_signals.py b/src/framevitals/dataset_signals.py index 6782e20..8f25a1d 100644 --- a/src/framevitals/dataset_signals.py +++ b/src/framevitals/dataset_signals.py @@ -1,81 +1,68 @@ """ Dataset Signal Detector ======================== -Produces a flat dictionary of boolean + numeric signals describing -the structural characteristics of the dataset. +Produces a flat dictionary of boolean + numeric signals describing the +structural characteristics of the dataset. -These signals drive the analysis selector engine — they answer -questions like "does this dataset have missing values?" or -"are there ID-like columns?" without hardcoding domain logic. +Signals reuse the role map and profile when available so semantic/type scans +and numeric correlation work are performed once per pipeline run. """ -import re - -import pandas as pd - from framevitals.column_roles import ( get_columns_with_role, - get_meaningful_numeric_columns, infer_column_roles, ) -_EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") -_TEXT_DTYPES = ["object", "string", "category"] - - -def _detect_long_text_columns(df: pd.DataFrame) -> list: - result = [] - for col in df.select_dtypes(include=_TEXT_DTYPES).columns: - lengths = df[col].dropna().astype(str).str.len() - if len(lengths) == 0: - continue - if float(lengths.mean()) > 50 or int(lengths.max()) > 200: - result.append(col) - return result +def _detect_potential_leakage(profile: dict, column_roles: dict) -> tuple[bool, bool]: + """Inspect cached profiler correlations for very high non-ID relationships.""" + correlations = profile.get("correlations", {}) or {} + correlation_metadata = profile.get("correlation_metadata", {}) or {} + truncated = bool(correlation_metadata.get("truncated", False)) + excluded = {"id_like", "time_like", "sequence_like", "constant"} + meaningful = { + column + for column, info in column_roles.items() + if info.get("is_numeric") and not set(info.get("roles", [])).intersection(excluded) + } -def _detect_email_columns(df: pd.DataFrame) -> list: - result = [] - for col in df.select_dtypes(include=_TEXT_DTYPES).columns: - sample = df[col].dropna().astype(str).head(100) - if sample.empty: + for left, values in correlations.items(): + if left not in meaningful or not isinstance(values, dict): continue - match_ratio = sample.apply(lambda v: bool(_EMAIL_PATTERN.match(v))).mean() - if match_ratio >= 0.5: - result.append(col) - return result - - -def _detect_potential_leakage(df: pd.DataFrame, column_roles: dict) -> bool: - """Return whether a very high non-ID numeric correlation suggests leakage.""" - meaningful = get_meaningful_numeric_columns(df, column_roles) - if len(meaningful) < 2: - return False - try: - corr = df[meaningful].corr(numeric_only=True).abs() - for i, a in enumerate(meaningful): - for b in meaningful[i + 1 :]: - val = corr.loc[a, b] - if pd.notna(val) and val >= 0.98: - return True - except Exception: - pass - return False + for right, value in values.items(): + if right == left or right not in meaningful or value is None: + continue + try: + if abs(float(value)) >= 0.98: + return True, truncated + except (TypeError, ValueError): + continue + return False, truncated def detect_dataset_signals( - df: pd.DataFrame, + df, profile: dict, column_roles: dict | None = None, + source_shape: tuple[int, int] | None = None, ) -> dict: """Produce structural signals, reusing cached pipeline metadata when supplied. ``column_roles`` is optional for backward compatibility. The main analysis pipeline passes its already-computed role map so this stage does not repeat - the most expensive per-column semantic scan. + per-column semantic scans. + + ``source_shape`` lets streaming/planning callers infer value-based roles on + a bounded row sample while preserving true full-dataset row/column counts + for scale signals and missingness percentages. """ - rows, cols = df.shape + if source_shape is None: + rows, cols = df.shape + else: + rows, cols = (int(source_shape[0]), int(source_shape[1])) + if rows < 0 or cols < 0: + raise ValueError("source_shape values must be non-negative.") numeric_cols = profile.get("numeric_columns", []) categorical_cols = profile.get("categorical_columns", []) @@ -108,16 +95,26 @@ def detect_dataset_signals( high_card = get_columns_with_role(column_roles, "high_cardinality") unique_like = get_columns_with_role(column_roles, "unique_like") target_candidates = get_columns_with_role(column_roles, "target_candidate") - - # Column-role inference already computes the same full-column text-length - # rule, so reuse it instead of scanning text columns a second time. long_text = get_columns_with_role(column_roles, "long_text") - email_like = _detect_email_columns(df) - has_leakage_risk = _detect_potential_leakage(df, column_roles) - lower_map = {c.lower(): c for c in df.columns} - has_bid_ask = "bid" in lower_map and "ask" in lower_map + email_like = get_columns_with_role(column_roles, "email_like") + url_like = get_columns_with_role(column_roles, "url_like") + uuid_like = get_columns_with_role(column_roles, "uuid_like") + ip_address_like = get_columns_with_role(column_roles, "ip_address_like") + phone_like = get_columns_with_role(column_roles, "phone_like") + percentage_like = get_columns_with_role(column_roles, "percentage_like") + currency_like = get_columns_with_role(column_roles, "currency_like") + json_like = get_columns_with_role(column_roles, "json_like") + boolean_token_like = get_columns_with_role(column_roles, "boolean_token_like") + + has_leakage_risk, leakage_scan_truncated = _detect_potential_leakage( + profile, + column_roles, + ) + source_columns = profile.get("columns", list(df.columns)) + lower_map = {str(column).lower(): column for column in source_columns} + has_bid_ask = "bid" in lower_map and "ask" in lower_map has_time_series = len(time_like) > 0 and len(numeric_cols) >= 1 and rows >= 20 return { @@ -149,7 +146,16 @@ def detect_dataset_signals( "has_time_series_structure": has_time_series, "has_sensitive_column_candidates": len(sensitive) > 0, "has_email_like_columns": len(email_like) > 0, + "has_url_like_columns": len(url_like) > 0, + "has_uuid_like_columns": len(uuid_like) > 0, + "has_ip_address_like_columns": len(ip_address_like) > 0, + "has_phone_like_columns": len(phone_like) > 0, + "has_percentage_like_columns": len(percentage_like) > 0, + "has_currency_like_columns": len(currency_like) > 0, + "has_json_like_columns": len(json_like) > 0, + "has_boolean_token_like_columns": len(boolean_token_like) > 0, "has_potential_leakage": has_leakage_risk, + "leakage_scan_truncated": leakage_scan_truncated, "has_target_candidates": len(target_candidates) > 0, "is_small_dataset": rows < 200, "is_large_dataset": rows >= 100000, @@ -163,6 +169,14 @@ def detect_dataset_signals( "sensitive_columns": sensitive, "constant_columns": constant, "email_like_columns": email_like, + "url_like_columns": url_like, + "uuid_like_columns": uuid_like, + "ip_address_like_columns": ip_address_like, + "phone_like_columns": phone_like, + "percentage_like_columns": percentage_like, + "currency_like_columns": currency_like, + "json_like_columns": json_like, + "boolean_token_like_columns": boolean_token_like, "long_text_columns": long_text, "binary_columns": binary, "low_cardinality_columns": low_card, diff --git a/src/framevitals/deep_statistics_v2.py b/src/framevitals/deep_statistics_v2.py index 9c3c446..0c9576b 100644 --- a/src/framevitals/deep_statistics_v2.py +++ b/src/framevitals/deep_statistics_v2.py @@ -114,6 +114,58 @@ def _outlier_flags(series: pd.Series) -> dict: return {"iqr": iqr_count, "z3": z3_count, "mad_z": mad_count, "n": n} +def _legacy_anderson_critical_5pct(result: Any) -> float | int | None: + """Extract the legacy 5% critical value without assuming a fixed table index.""" + critical_values = getattr(result, "critical_values", None) + significance_levels = getattr(result, "significance_level", None) + if critical_values is None: + return None + + if significance_levels is not None: + try: + levels = np.asarray(significance_levels, dtype=float) + values = np.asarray(critical_values, dtype=float) + if levels.size and values.size == levels.size: + index = int(np.argmin(np.abs(levels - 5.0))) + return _safe_float(values[index]) + except (TypeError, ValueError): + pass + + try: + return _safe_float(critical_values[2]) + except (IndexError, TypeError): + return None + + +def _anderson_normality(series: pd.Series) -> dict[str, Any]: + """Run Anderson-Darling using the modern p-value API when available. + + SciPy 1.17 introduced ``method`` and deprecated the legacy critical-value + result attributes. FrameVitals supports older SciPy releases too, so this + helper opts into the modern interpolated p-value and falls back only when + the installed SciPy does not recognize the keyword. + """ + try: + result = stats.anderson(series, dist="norm", method="interpolate") + except TypeError as exc: + if "method" not in str(exc): + raise + result = stats.anderson(series, dist="norm") + return { + "statistic": _safe_float(result.statistic), + "p_value": None, + "critical_5pct": _legacy_anderson_critical_5pct(result), + "method": "legacy_critical_values", + } + + return { + "statistic": _safe_float(result.statistic), + "p_value": _safe_float(result.pvalue), + "critical_5pct": None, + "method": "interpolate", + } + + def _normality(series: pd.Series) -> dict: """Run multiple normality tests, prefer Shapiro for small n, D'Agostino for medium.""" s = series.dropna() @@ -138,13 +190,9 @@ def _normality(series: pd.Series) -> dict: except Exception as exc: out["dagostino"] = {"error": str(exc)} - # Anderson-Darling + # Anderson-Darling. Explicit p-value method avoids SciPy >=1.17 deprecation warnings. try: - result = stats.anderson(s, dist="norm") - out["anderson"] = { - "statistic": _safe_float(result.statistic), - "critical_5pct": _safe_float(result.critical_values[2]), - } + out["anderson"] = _anderson_normality(s) except Exception as exc: out["anderson"] = {"error": str(exc)} diff --git a/src/framevitals/deep_triage.py b/src/framevitals/deep_triage.py new file mode 100644 index 0000000..8921581 --- /dev/null +++ b/src/framevitals/deep_triage.py @@ -0,0 +1,166 @@ +"""Cheap column triage for expensive deep-analysis routines. + +The profiler already describes every column. Deep statistical routines therefore +should not blindly spend bootstrap/distribution-fitting work on every feature. +This module scores all candidate columns with inexpensive vectorized statistics +and returns a stable, bounded subset for heavyweight diagnostics. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + + +_MODE_LIMITS: dict[str, tuple[int, int]] = { + "quick": (4, 4), + "standard": (8, 6), + "deep": (12, 8), + "research": (40, 20), +} + + +@dataclass(frozen=True, slots=True) +class DeepTriageResult: + selected_numeric: tuple[str, ...] + selected_categorical: tuple[str, ...] + numeric_scores: dict[str, float] + categorical_scores: dict[str, float] + numeric_available: int + categorical_available: int + numeric_limit: int + categorical_limit: int + + @property + def selected_columns(self) -> tuple[str, ...]: + return self.selected_numeric + self.selected_categorical + + def to_dict(self) -> dict[str, Any]: + return { + "method": "vectorized_interest_ranking", + "selected_numeric": list(self.selected_numeric), + "selected_categorical": list(self.selected_categorical), + "numeric_available": self.numeric_available, + "categorical_available": self.categorical_available, + "numeric_limit": self.numeric_limit, + "categorical_limit": self.categorical_limit, + "numeric_truncated": self.numeric_available > len(self.selected_numeric), + "categorical_truncated": self.categorical_available > len(self.selected_categorical), + "numeric_scores": dict(self.numeric_scores), + "categorical_scores": dict(self.categorical_scores), + } + + +def _finite_score(value: Any) -> float: + try: + score = float(value) + except (TypeError, ValueError): + return 0.0 + return score if np.isfinite(score) else 0.0 + + +def _rank_scores(scores: dict[str, float], original_order: list[str], limit: int) -> tuple[str, ...]: + order = {column: index for index, column in enumerate(original_order)} + ranked = sorted( + original_order, + key=lambda column: (-_finite_score(scores.get(column)), order[column]), + ) + return tuple(ranked[: max(0, min(limit, len(ranked)))]) + + +def _numeric_interest_scores(frame: pd.DataFrame) -> dict[str, float]: + if frame.empty or frame.shape[1] == 0: + return {} + + numeric = frame.replace([np.inf, -np.inf], np.nan) + n_rows = max(len(numeric), 1) + missing = numeric.isna().mean() + unique = numeric.nunique(dropna=True) + + with np.errstate(all="ignore"): + skew = numeric.skew(axis=0, skipna=True).abs().fillna(0.0) + kurtosis = numeric.kurt(axis=0, skipna=True).abs().fillna(0.0) + std = numeric.std(axis=0, skipna=True).fillna(0.0) + + scores: dict[str, float] = {} + for column in numeric.columns: + unique_count = int(unique.get(column, 0)) + unique_ratio = unique_count / n_rows + near_constant = 1.0 if unique_count <= 2 or _finite_score(std.get(column)) <= 1e-12 else 0.0 + score = ( + min(_finite_score(skew.get(column)), 8.0) * 1.5 + + min(_finite_score(kurtosis.get(column)), 20.0) * 0.65 + + min(_finite_score(missing.get(column)), 1.0) * 5.0 + + near_constant * 2.5 + + min(unique_ratio, 1.0) * 0.35 + ) + scores[str(column)] = round(float(score), 6) + return scores + + +def _categorical_interest_scores(frame: pd.DataFrame) -> dict[str, float]: + if frame.empty or frame.shape[1] == 0: + return {} + + n_rows = max(len(frame), 1) + missing = frame.isna().mean() + unique = frame.nunique(dropna=True) + scores: dict[str, float] = {} + + for column in frame.columns: + cardinality = int(unique.get(column, 0)) + missing_rate = min(_finite_score(missing.get(column)), 1.0) + if cardinality <= 1: + relationship_value = 0.5 + elif cardinality == 2: + relationship_value = 3.0 + elif cardinality <= 12: + relationship_value = 2.5 + elif cardinality <= 30: + relationship_value = 1.5 + else: + relationship_value = 0.25 + rarity_signal = min(cardinality / n_rows, 1.0) * 0.25 + score = missing_rate * 8.0 + relationship_value + rarity_signal + scores[str(column)] = round(float(score), 6) + return scores + + +def triage_deep_columns(dataframe: pd.DataFrame, *, mode: str) -> DeepTriageResult: + """Rank all deep-statistics candidates and return a bounded stable subset.""" + if mode not in _MODE_LIMITS: + raise ValueError(f"Unknown analysis mode: {mode}") + + numeric_columns = dataframe.select_dtypes(include=[np.number]).columns.tolist() + categorical_columns = dataframe.select_dtypes( + include=["object", "category", "bool", "string"], + ).columns.tolist() + numeric_limit, categorical_limit = _MODE_LIMITS[mode] + + numeric_scores = _numeric_interest_scores(dataframe[numeric_columns]) if numeric_columns else {} + categorical_scores = ( + _categorical_interest_scores(dataframe[categorical_columns]) + if categorical_columns + else {} + ) + + selected_numeric = _rank_scores(numeric_scores, numeric_columns, numeric_limit) + selected_categorical = _rank_scores( + categorical_scores, + categorical_columns, + categorical_limit, + ) + + return DeepTriageResult( + selected_numeric=selected_numeric, + selected_categorical=selected_categorical, + numeric_scores=numeric_scores, + categorical_scores=categorical_scores, + numeric_available=len(numeric_columns), + categorical_available=len(categorical_columns), + numeric_limit=numeric_limit, + categorical_limit=categorical_limit, + ) diff --git a/src/framevitals/drift_analysis.py b/src/framevitals/drift_analysis.py index f5eecfb..b6660fc 100644 --- a/src/framevitals/drift_analysis.py +++ b/src/framevitals/drift_analysis.py @@ -1,47 +1,38 @@ -""" -Drift / Compare Analysis (WS-7) -================================ -Compare two dataframes (a "reference" and a "current") column by column and -quantify how much each column's distribution has shifted. - -Tests: - Numeric columns: - - Population Stability Index (PSI), 10 quantile bins - - Kolmogorov-Smirnov two-sample test (ks_2samp) - - Mean shift in standard deviations (z-shift) - Categorical columns: - - PSI on category proportions - - Chi-square test of independence - -Severity buckets (PSI): - < 0.10 -> stable - < 0.25 -> minor - < 0.50 -> moderate - >= 0.50 -> severe - -Public entry points: - compare_datasets(df_a, df_b, columns=None) -> dict - split_by_date(df, date_column, ratio=0.5) -> tuple[DataFrame, DataFrame] - -Output is fully JSON-safe. +"""Dataset drift and change analysis. + +FrameVitals compares a reference and current dataframe across schema, types, +missingness, and value distributions. PSI remains available for compatibility, +but the public verdict now combines multiple interpretable signals rather than +letting one statistic decide the entire result. """ from __future__ import annotations import math import warnings +from collections import Counter from typing import Any import numpy as np import pandas as pd from scipy import stats +from scipy.spatial.distance import jensenshannon + + +_SEVERITY_ORDER = { + "unknown": -1, + "stable": 0, + "minor": 1, + "moderate": 2, + "severe": 3, +} def _safe_float(value: Any, ndigits: int = 4) -> float | int | None: if value is None: return None try: - if isinstance(value, (np.integer,)): + if isinstance(value, np.integer): return int(value) if isinstance(value, (np.floating, float)): v = float(value) @@ -67,13 +58,90 @@ def _classify_psi(psi: float | None) -> str: return "severe" +def _classify_missingness(delta_percentage_points: float) -> str: + delta = abs(delta_percentage_points) + if delta < 2: + return "stable" + if delta < 5: + return "minor" + if delta < 15: + return "moderate" + return "severe" + + +def _classify_numeric_distance(value: float | None) -> str: + if value is None: + return "unknown" + if value < 0.10: + return "stable" + if value < 0.25: + return "minor" + if value < 0.50: + return "moderate" + return "severe" + + +def _classify_js_distance(value: float | None) -> str: + if value is None: + return "unknown" + if value < 0.05: + return "stable" + if value < 0.10: + return "minor" + if value < 0.25: + return "moderate" + return "severe" + + +def _max_severity(*values: str) -> str: + usable = [value for value in values if value in _SEVERITY_ORDER] + if not usable: + return "unknown" + return max(usable, key=lambda value: _SEVERITY_ORDER[value]) + + +def severity_at_least(actual: str, threshold: str) -> bool: + """Return whether a drift severity meets or exceeds ``threshold``.""" + if threshold not in {"minor", "moderate", "severe"}: + raise ValueError("threshold must be one of: minor, moderate, severe.") + return _SEVERITY_ORDER.get(actual, -1) >= _SEVERITY_ORDER[threshold] + + +def _dtype_family(series: pd.Series) -> str: + if pd.api.types.is_bool_dtype(series): + return "boolean" + if pd.api.types.is_datetime64_any_dtype(series): + return "datetime" + if pd.api.types.is_numeric_dtype(series): + return "numeric" + if ( + pd.api.types.is_object_dtype(series) + or pd.api.types.is_string_dtype(series.dtype) + or isinstance(series.dtype, pd.CategoricalDtype) + ): + return "categorical" + return "unsupported" + + def _shared_columns(df_a: pd.DataFrame, df_b: pd.DataFrame) -> list[str]: other_columns = set(df_b.columns) return [column for column in df_a.columns if column in other_columns] +def _missingness_metrics(ref: pd.Series, cur: pd.Series) -> dict[str, Any]: + ref_pct = float(ref.isna().mean() * 100) if len(ref) else 0.0 + cur_pct = float(cur.isna().mean() * 100) if len(cur) else 0.0 + delta = cur_pct - ref_pct + return { + "ref_missing_percent": _safe_float(ref_pct), + "cur_missing_percent": _safe_float(cur_pct), + "missingness_delta_percentage_points": _safe_float(delta), + "missingness_severity": _classify_missingness(delta), + } + + def _psi_numeric(ref: np.ndarray, cur: np.ndarray, bins: int = 10) -> float | None: - """Population Stability Index using quantile bins from `ref`.""" + """Population Stability Index using quantile bins from ``ref``.""" ref = ref[np.isfinite(ref)] cur = cur[np.isfinite(cur)] if len(ref) < 10 or len(cur) < 10: @@ -88,10 +156,8 @@ def _psi_numeric(ref: np.ndarray, cur: np.ndarray, bins: int = 10) -> float | No ref_counts, _ = np.histogram(ref, bins=edges) cur_counts, _ = np.histogram(cur, bins=edges) - ref_props = ref_counts / max(ref_counts.sum(), 1) cur_props = cur_counts / max(cur_counts.sum(), 1) - ref_props = np.where(ref_props == 0, 1e-6, ref_props) cur_props = np.where(cur_props == 0, 1e-6, cur_props) @@ -99,20 +165,36 @@ def _psi_numeric(ref: np.ndarray, cur: np.ndarray, bins: int = 10) -> float | No return psi if math.isfinite(psi) else None -def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict: - # Force float64 — pd.to_numeric leaves boolean dtype alone, and newer - # numpy refuses to subtract bool arrays inside np.quantile. +def _normalized_wasserstein(ref: np.ndarray, cur: np.ndarray) -> float | None: + if len(ref) < 2 or len(cur) < 2: + return None + distance = float(stats.wasserstein_distance(ref, cur)) + q1, q3 = np.quantile(ref, [0.25, 0.75]) + scale = float(q3 - q1) + if scale <= 0: + scale = float(np.std(ref, ddof=0)) + if scale <= 0: + scale = max(abs(float(np.mean(ref))), 1.0) + value = distance / scale + return value if math.isfinite(value) else None + + +def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict[str, Any]: + missingness = _missingness_metrics(ref, cur) ref_arr = pd.to_numeric(ref, errors="coerce").astype("float64").to_numpy() cur_arr = pd.to_numeric(cur, errors="coerce").astype("float64").to_numpy() ref_arr = ref_arr[np.isfinite(ref_arr)] cur_arr = cur_arr[np.isfinite(cur_arr)] if len(ref_arr) < 10 or len(cur_arr) < 10: + severity = missingness["missingness_severity"] return { "column": name, "type": "numeric", "available": False, "reason": "n<10 in one side", + "drift_severity": severity, + **missingness, } psi = _psi_numeric(ref_arr, cur_arr) @@ -133,22 +215,25 @@ def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict: pooled_std = ref_std if ref_std > 0 else cur_std if cur_std > 0 else 1.0 z_shift = float((cur_mean - ref_mean) / pooled_std) if pooled_std > 0 else None - severity = _classify_psi(psi) - if ks_p is not None and ks_p < 0.01 and severity == "stable": - severity = "minor" - - edges = np.linspace( - min(ref_arr.min(), cur_arr.min()), - max(ref_arr.max(), cur_arr.max()), - 21, + wasserstein = _normalized_wasserstein(ref_arr, cur_arr) + psi_severity = _classify_psi(psi) + wasserstein_severity = _classify_numeric_distance(wasserstein) + ks_severity = "minor" if ks_p is not None and ks_p < 0.01 else "stable" + severity = _max_severity( + psi_severity, + wasserstein_severity, + ks_severity, + missingness["missingness_severity"], ) + + combined_min = float(min(ref_arr.min(), cur_arr.min())) + combined_max = float(max(ref_arr.max(), cur_arr.max())) + if combined_min == combined_max: + combined_min -= 0.5 + combined_max += 0.5 + edges = np.linspace(combined_min, combined_max, 21) ref_hist, _ = np.histogram(ref_arr, bins=edges) cur_hist, _ = np.histogram(cur_arr, bins=edges) - histogram = { - "edges": [_safe_float(v) for v in edges.tolist()], - "ref": [int(v) for v in ref_hist.tolist()], - "cur": [int(v) for v in cur_hist.tolist()], - } return { "column": name, @@ -157,7 +242,9 @@ def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict: "n_ref": int(len(ref_arr)), "n_cur": int(len(cur_arr)), "psi": _safe_float(psi), - "psi_severity": severity, + "psi_severity": psi_severity, + "wasserstein_normalized": _safe_float(wasserstein), + "wasserstein_severity": wasserstein_severity, "ks_statistic": ks_stat, "ks_p_value": ks_p, "ks_significant": bool(ks_p is not None and ks_p < 0.01), @@ -166,25 +253,30 @@ def _numeric_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict: "ref_std": _safe_float(ref_std), "cur_std": _safe_float(cur_std), "z_shift": _safe_float(z_shift), - "histogram": histogram, + "drift_severity": severity, + **missingness, + "histogram": { + "edges": [_safe_float(value) for value in edges.tolist()], + "ref": [int(value) for value in ref_hist.tolist()], + "cur": [int(value) for value in cur_hist.tolist()], + }, } def _psi_categorical( ref: pd.Series, cur: pd.Series, -) -> tuple[float | None, list[str], dict, pd.Series, pd.Series]: - """Calculate categorical PSI and keep counts for downstream chi-square use.""" +) -> tuple[float | None, list[str], dict[str, Any], pd.Series, pd.Series, np.ndarray, np.ndarray]: ref_counts = ref.astype(str).value_counts(dropna=False) cur_counts = cur.astype(str).value_counts(dropna=False) - categories = sorted(set(ref_counts.index) | set(cur_counts.index)) + combined = ref_counts.add(cur_counts, fill_value=0).sort_values(ascending=False) + categories = [str(value) for value in combined.index.tolist()] if len(categories) < 2: - return None, categories, {}, ref_counts, cur_counts + return None, categories, {}, ref_counts, cur_counts, np.array([]), np.array([]) ref_total = max(int(ref_counts.sum()), 1) cur_total = max(int(cur_counts.sum()), 1) - ref_props = np.array([ref_counts.get(c, 0) / ref_total for c in categories], dtype=float) cur_props = np.array([cur_counts.get(c, 0) / cur_total for c in categories], dtype=float) @@ -199,40 +291,59 @@ def _psi_categorical( "ref_props": [round(float(p), 4) for p in ref_props[:30]], "cur_props": [round(float(p), 4) for p in cur_props[:30]], } - return ( psi if math.isfinite(psi) else None, categories, distribution, ref_counts, cur_counts, + ref_props, + cur_props, ) -def _categorical_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict: +def _categorical_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict[str, Any]: + missingness = _missingness_metrics(ref, cur) ref_clean = ref.dropna() cur_clean = cur.dropna() if len(ref_clean) < 10 or len(cur_clean) < 10: + severity = missingness["missingness_severity"] return { "column": name, "type": "categorical", "available": False, "reason": "n<10 in one side", + "drift_severity": severity, + **missingness, } - psi, categories, distribution, ref_counts, cur_counts = _psi_categorical( - ref_clean, - cur_clean, - ) + ( + psi, + categories, + distribution, + ref_counts, + cur_counts, + ref_props, + cur_props, + ) = _psi_categorical(ref_clean, cur_clean) if psi is None: return { "column": name, "type": "categorical", "available": False, "reason": "single category", + "drift_severity": missingness["missingness_severity"], + **missingness, } + js_distance = None + if len(ref_props) and len(cur_props): + try: + js_distance = float(jensenshannon(ref_props, cur_props, base=2)) + except Exception: + js_distance = None + chi2_stat, chi2_p, dof = None, None, None try: top_categories = categories[:30] @@ -254,6 +365,16 @@ def _categorical_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict ref_unique_set = set(ref_unique) cur_unique_set = set(cur_unique) + psi_severity = _classify_psi(psi) + js_severity = _classify_js_distance(js_distance) + chi_severity = "minor" if chi2_p is not None and chi2_p < 0.01 else "stable" + severity = _max_severity( + psi_severity, + js_severity, + chi_severity, + missingness["missingness_severity"], + ) + return { "column": name, "type": "categorical", @@ -261,76 +382,187 @@ def _categorical_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict "n_ref": int(len(ref_clean)), "n_cur": int(len(cur_clean)), "psi": _safe_float(psi), - "psi_severity": _classify_psi(psi), + "psi_severity": psi_severity, + "jensen_shannon_distance": _safe_float(js_distance), + "jensen_shannon_severity": js_severity, "chi2_statistic": _safe_float(chi2_stat), "chi2_p_value": _safe_float(chi2_p), "chi2_dof": int(dof) if dof is not None else None, "chi2_significant": bool(chi2_p is not None and chi2_p < 0.01), "n_categories_ref": int(len(ref_unique)), "n_categories_cur": int(len(cur_unique)), - "new_categories": [c for c in cur_unique if c not in ref_unique_set][:10], - "missing_categories": [c for c in ref_unique if c not in cur_unique_set][:10], + "new_categories": [value for value in cur_unique if value not in ref_unique_set][:10], + "missing_categories": [value for value in ref_unique if value not in cur_unique_set][:10], + "drift_severity": severity, + **missingness, "distribution": distribution, } +def _datetime_column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict[str, Any]: + missingness = _missingness_metrics(ref, cur) + ref_dt = pd.to_datetime(ref, errors="coerce", utc=True).dropna() + cur_dt = pd.to_datetime(cur, errors="coerce", utc=True).dropna() + if len(ref_dt) < 10 or len(cur_dt) < 10: + return { + "column": name, + "type": "datetime", + "available": False, + "reason": "n<10 in one side", + "drift_severity": missingness["missingness_severity"], + **missingness, + } + + ref_days = ref_dt.astype("int64").to_numpy(dtype="float64") / 86_400_000_000_000 + cur_days = cur_dt.astype("int64").to_numpy(dtype="float64") / 86_400_000_000_000 + psi = _psi_numeric(ref_days, cur_days) + wasserstein = _normalized_wasserstein(ref_days, cur_days) + try: + ks_stat, ks_p = stats.ks_2samp(ref_days, cur_days) + except Exception: + ks_stat, ks_p = None, None + + psi_severity = _classify_psi(psi) + wasserstein_severity = _classify_numeric_distance(wasserstein) + ks_severity = "minor" if ks_p is not None and ks_p < 0.01 else "stable" + severity = _max_severity( + psi_severity, + wasserstein_severity, + ks_severity, + missingness["missingness_severity"], + ) + + return { + "column": name, + "type": "datetime", + "available": True, + "n_ref": int(len(ref_dt)), + "n_cur": int(len(cur_dt)), + "psi": _safe_float(psi), + "psi_severity": psi_severity, + "wasserstein_normalized": _safe_float(wasserstein), + "wasserstein_severity": wasserstein_severity, + "ks_statistic": _safe_float(ks_stat), + "ks_p_value": _safe_float(ks_p), + "ks_significant": bool(ks_p is not None and ks_p < 0.01), + "ref_min": ref_dt.min().isoformat(), + "ref_max": ref_dt.max().isoformat(), + "cur_min": cur_dt.min().isoformat(), + "cur_max": cur_dt.max().isoformat(), + "drift_severity": severity, + **missingness, + } + + +def _column_drift(name: str, ref: pd.Series, cur: pd.Series) -> dict[str, Any]: + ref_family = _dtype_family(ref) + cur_family = _dtype_family(cur) + if ref_family != cur_family: + return { + "column": name, + "type": "type_mismatch", + "available": False, + "reason": f"Type changed from {ref_family} to {cur_family}.", + "ref_type": ref_family, + "cur_type": cur_family, + "drift_severity": "severe", + **_missingness_metrics(ref, cur), + } + if ref_family == "numeric": + return _numeric_column_drift(name, ref, cur) + if ref_family in {"categorical", "boolean"}: + return _categorical_column_drift(name, ref, cur) + if ref_family == "datetime": + return _datetime_column_drift(name, ref, cur) + return { + "column": name, + "type": "unsupported", + "available": False, + "reason": f"Unsupported dtype: {ref.dtype}", + "drift_severity": "unknown", + **_missingness_metrics(ref, cur), + } + + def compare_datasets( df_ref: pd.DataFrame, df_cur: pd.DataFrame, columns: list[str] | None = None, max_columns: int = 30, -) -> dict: - """Compare two dataframes column by column and return a drift report.""" - shared = _shared_columns(df_ref, df_cur) +) -> dict[str, Any]: + """Compare two dataframes and return a structured drift/change report.""" + if max_columns < 1: + raise ValueError("max_columns must be at least 1.") + + ref_columns = set(df_ref.columns) + cur_columns = set(df_cur.columns) + all_shared = _shared_columns(df_ref, df_cur) + requested_missing_ref: list[str] = [] + requested_missing_cur: list[str] = [] + + shared = all_shared if columns: - requested = set(columns) - shared = [column for column in shared if column in requested] + requested = list(dict.fromkeys(columns)) + requested_set = set(requested) + requested_missing_ref = [name for name in requested if name not in ref_columns] + requested_missing_cur = [name for name in requested if name not in cur_columns] + shared = [name for name in all_shared if name in requested_set] + + total_selected = len(shared) + truncated = total_selected > max_columns shared = shared[:max_columns] + added_columns = sorted(cur_columns - ref_columns) + removed_columns = sorted(ref_columns - cur_columns) + dtype_changes = [ + { + "column": name, + "reference_type": _dtype_family(df_ref[name]), + "current_type": _dtype_family(df_cur[name]), + } + for name in all_shared + if _dtype_family(df_ref[name]) != _dtype_family(df_cur[name]) + ] + + row_delta_pct = None + if len(df_ref): + row_delta_pct = (len(df_cur) - len(df_ref)) / len(df_ref) * 100 + if not shared: + schema_severity = "severe" if removed_columns or dtype_changes else "moderate" if added_columns else "stable" return { "available": False, - "reason": "No shared columns between the two datasets.", + "reason": "No shared selected columns between the two datasets.", "ref_shape": list(df_ref.shape), "cur_shape": list(df_cur.shape), + "schema": { + "added_columns": added_columns, + "removed_columns": removed_columns, + "dtype_changes": dtype_changes, + "requested_missing_in_reference": requested_missing_ref, + "requested_missing_in_current": requested_missing_cur, + "severity": schema_severity, + }, } - column_results: list[dict] = [] - for col in shared: - ref_series = df_ref[col] - cur_series = df_cur[col] - - if pd.api.types.is_numeric_dtype(ref_series) and pd.api.types.is_numeric_dtype( - cur_series - ): - column_results.append(_numeric_column_drift(col, ref_series, cur_series)) - elif ( - pd.api.types.is_object_dtype(ref_series) - or pd.api.types.is_string_dtype(ref_series.dtype) - or isinstance(ref_series.dtype, pd.CategoricalDtype) - or pd.api.types.is_bool_dtype(ref_series) - ): - column_results.append(_categorical_column_drift(col, ref_series, cur_series)) - else: - column_results.append( - { - "column": col, - "type": "unsupported", - "available": False, - "reason": f"Unsupported dtype: {ref_series.dtype}", - } - ) - - severity_counts = { - "stable": 0, - "minor": 0, - "moderate": 0, - "severe": 0, - "unknown": 0, - } - for entry in column_results: - sev = entry.get("psi_severity") if entry.get("available") else "unknown" - severity_counts[sev] = severity_counts.get(sev, 0) + 1 + column_results = [_column_drift(name, df_ref[name], df_cur[name]) for name in shared] + + severity_counts = Counter( + entry.get("drift_severity", "unknown") for entry in column_results + ) + for label in _SEVERITY_ORDER: + severity_counts.setdefault(label, 0) + + schema_severity = "stable" + if removed_columns or dtype_changes: + schema_severity = "severe" + elif added_columns: + schema_severity = "moderate" + + column_severity = _max_severity( + *(entry.get("drift_severity", "unknown") for entry in column_results) + ) + overall_verdict = _max_severity(column_severity, schema_severity) severity_rank = { "severe": 0, @@ -340,35 +572,76 @@ def compare_datasets( "unknown": 4, } - def _rank(entry): - sev = entry.get("psi_severity") if entry.get("available") else "unknown" + def _rank(entry: dict[str, Any]) -> tuple[int, float]: + severity = entry.get("drift_severity", "unknown") psi = entry.get("psi") - return (severity_rank.get(sev, 9), -(psi if isinstance(psi, (int, float)) else 0)) + return ( + severity_rank.get(severity, 9), + -(psi if isinstance(psi, (int, float)) else 0), + ) column_results.sort(key=_rank) - if severity_counts["severe"] > 0: - verdict = "severe" - elif severity_counts["moderate"] > 0: - verdict = "moderate" - elif severity_counts["minor"] > 0: - verdict = "minor" + gate_reasons: list[str] = [] + if removed_columns: + gate_reasons.append(f"{len(removed_columns)} reference columns are missing from current data.") + if dtype_changes: + gate_reasons.append(f"{len(dtype_changes)} shared columns changed type family.") + if added_columns: + gate_reasons.append(f"{len(added_columns)} new columns appeared in current data.") + severe_columns = [ + entry["column"] for entry in column_results if entry.get("drift_severity") == "severe" + ] + moderate_columns = [ + entry["column"] for entry in column_results if entry.get("drift_severity") == "moderate" + ] + if severe_columns: + gate_reasons.append(f"Severe drift detected in: {', '.join(severe_columns[:10])}.") + elif moderate_columns: + gate_reasons.append(f"Moderate drift detected in: {', '.join(moderate_columns[:10])}.") + + if overall_verdict == "severe": + gate_status = "fail" + elif overall_verdict in {"minor", "moderate"}: + gate_status = "warn" else: - verdict = "stable" + gate_status = "pass" return { "available": True, "ref_shape": list(df_ref.shape), "cur_shape": list(df_cur.shape), + "row_count_change_percent": _safe_float(row_delta_pct), "shared_columns": shared, + "selection": { + "total_shared_columns": len(all_shared), + "total_selected_columns": total_selected, + "max_columns": int(max_columns), + "truncated": truncated, + "requested_missing_in_reference": requested_missing_ref, + "requested_missing_in_current": requested_missing_cur, + }, + "schema": { + "added_columns": added_columns, + "removed_columns": removed_columns, + "dtype_changes": dtype_changes, + "severity": schema_severity, + }, "summary": { "n_columns_compared": len(column_results), - "severity_counts": severity_counts, - "overall_verdict": verdict, + "n_columns_available": sum(bool(entry.get("available")) for entry in column_results), + "severity_counts": dict(severity_counts), + "overall_verdict": overall_verdict, + }, + "gate": { + "status": gate_status, + "severity": overall_verdict, + "reasons": gate_reasons, }, "interpretation": ( - "PSI < 0.10 stable · 0.10-0.25 minor · 0.25-0.50 moderate · ≥ 0.50 severe. " - "Numeric columns also report KS test; categorical columns report chi-square." + "Drift severity combines PSI with normalized Wasserstein distance for numeric/date columns, " + "Jensen-Shannon distance for categorical columns, statistical tests, missingness changes, " + "and structural schema/type changes." ), "columns": column_results, } @@ -382,6 +655,8 @@ def split_by_date( """Split a dataframe chronologically into earlier and later partitions.""" if date_column not in df.columns: raise ValueError(f"Column not found: {date_column}") + if not 0 < ratio < 1: + raise ValueError("ratio must be between 0 and 1.") parsed = pd.to_datetime(df[date_column], errors="coerce", format="mixed") if parsed.notna().mean() < 0.7: diff --git a/src/framevitals/duckdb_source.py b/src/framevitals/duckdb_source.py new file mode 100644 index 0000000..7a53343 --- /dev/null +++ b/src/framevitals/duckdb_source.py @@ -0,0 +1,118 @@ +"""Optional DuckDB relation adapter for FrameVitals source-aware execution.""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from dataclasses import dataclass, field +from typing import Any + +import pandas as pd + +from framevitals.sources import DatasetMetadata + + +def _quoted_identifier(name: str) -> str: + """Quote a DuckDB identifier without treating column names as expressions.""" + return '"' + str(name).replace('"', '""') + '"' + + +@dataclass(slots=True) +class DuckDBRelationSource: + """Stream a lazy DuckDB relation through Arrow record batches. + + ``inspect()`` executes an exact ``count(*)`` once and caches the result. + This can scan the underlying relation, but it does not materialize all rows + in Python memory. Row data is consumed later through ``to_arrow_reader``. + """ + + relation: Any + name: str = "" + _metadata_cache: DatasetMetadata | None = field(default=None, init=False, repr=False) + _schema_cache: Any = field(default=None, init=False, repr=False) + + def inspect(self) -> DatasetMetadata: + if self._metadata_cache is not None: + return self._metadata_cache + + columns = list(self.relation.columns) + count_row = self.relation.aggregate( + "count(*) AS __framevitals_rows" + ).fetchone() + if not count_row: + raise ValueError("DuckDB relation did not return a row count.") + + rows = int(count_row[0]) + schema = self.schema() + self._metadata_cache = DatasetMetadata( + name=self.name, + kind="relation", + format="duckdb", + rows=rows, + columns=len(columns), + size_bytes=None, + materialized=False, + supports_projection=True, + supports_streaming=True, + ) + if len(schema) != len(columns): + raise ValueError( + "DuckDB relation schema changed while FrameVitals inspected it." + ) + return self._metadata_cache + + def schema(self): + """Return an Arrow schema without fetching relation rows.""" + if self._schema_cache is None: + self._schema_cache = self.relation.limit(0).to_arrow_table().schema + return self._schema_cache + + def iter_batches( + self, + *, + batch_size: int = 65_536, + columns: Sequence[str] | None = None, + ) -> Iterator[Any]: + if batch_size < 1: + raise ValueError("batch_size must be at least 1.") + + projected = self.relation + if columns is not None: + available = set(self.relation.columns) + missing = [column for column in columns if column not in available] + if missing: + raise KeyError( + "DuckDB relation does not contain requested column(s): " + + ", ".join(map(str, missing)) + ) + projected = self.relation.select( + *[_quoted_identifier(column) for column in columns] + ) + + reader = projected.to_arrow_reader(int(batch_size)) + yield from reader + + def load(self) -> pd.DataFrame: + """Materialize the complete relation only for exact/full-row APIs.""" + dataframe = self.relation.df() + if dataframe.empty: + raise ValueError("Dataset is empty: ") + return dataframe + + +def resolve_duckdb_source(data: Any) -> DuckDBRelationSource | None: + """Recognize DuckDB relations without importing DuckDB for normal inputs.""" + data_type = type(data) + if data_type.__name__ != "DuckDBPyRelation": + return None + if not data_type.__module__.startswith(("duckdb", "_duckdb")): + return None + + try: + import duckdb + except ImportError: + return None + + relation_type = getattr(duckdb, "DuckDBPyRelation", None) + if relation_type is None or not isinstance(data, relation_type): + return None + return DuckDBRelationSource(data) diff --git a/src/framevitals/execution.py b/src/framevitals/execution.py new file mode 100644 index 0000000..1c951fa --- /dev/null +++ b/src/framevitals/execution.py @@ -0,0 +1,309 @@ +"""Adaptive execution budgets for FrameVitals analyses. + +The budget layer is intentionally backend-agnostic. It describes how much raw +work an analysis may attempt for a given dataset shape and analysis mode. Native +Rust, Arrow, CUDA, and distributed backends can consume the same contract later. + +The immediate goal is safety: expensive statistical routines must never infer +that a 100k-row input means they should allocate O(n^2) intermediates. Sampling +is deterministic and must be disclosed in result metadata. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + +import numpy as np +import pandas as pd + + +_VALID_MODES = {"quick", "standard", "deep", "research"} +_SAMPLE_SEED = 0x9E3779B97F4A7C15 + +# Full-stream profiling is valuable, but on ultra-wide sources scanning every +# cell defeats the purpose of streaming. These budgets cap the number of source +# cells inspected by the reusable profile pass while preserving the true source +# shape in execution metadata. +_STREAMING_PROFILE_CELL_BUDGETS = { + "quick": 64_000_000, + "standard": 96_000_000, + "deep": 128_000_000, + "research": 256_000_000, +} +_STREAMING_PROFILE_COLUMN_CAPS = { + "quick": 64, + "standard": 96, + "deep": 128, + "research": 256, +} + + +@dataclass(frozen=True, slots=True) +class ExecutionBudget: + """Resolved resource policy for one analysis run. + + Counts are upper bounds, not promises that every analysis consumes the full + allowance. Backends are encouraged to stop earlier when an estimate has + converged or an operation is not applicable. + """ + + mode: str + rows: int + columns: int + cells: int + scale_class: str + large_dataset: bool + wide_dataset: bool + ultra_wide_dataset: bool + quality_sample_rows: int + deep_statistics_sample_rows: int + bootstrap_sample_rows: int + distribution_sample_rows: int + pair_sample_rows: int + anomaly_sample_rows: int + time_series_sample_rows: int + relationship_pair_budget: int + max_memory_heavy_parallelism: int + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _bounded(requested: int, rows: int) -> int: + if rows <= 0: + return 0 + return min(int(requested), int(rows)) + + +def _deterministic_stratified_positions( + rows: int, + target_rows: int, + *, + seed: int = _SAMPLE_SEED, +) -> np.ndarray: + """Choose one deterministic pseudo-random row from each equal-width stratum. + + Fixed evenly spaced samples can lock onto periodic structure. Stratified jitter + retains deterministic whole-dataset coverage while breaking that phase locking. + Positions are returned sorted, so temporal order remains available to callers + that need it, without allocating a permutation proportional to the source size. + """ + rows = int(rows) + target_rows = int(target_rows) + count = min(rows, target_rows) + if count <= 0: + return np.empty(0, dtype=np.int64) + if count == rows: + return np.arange(rows, dtype=np.int64) + if count == 1: + return np.array([rows // 2], dtype=np.int64) + + edges = np.fromiter( + ((index * rows) // count for index in range(count + 1)), + dtype=np.int64, + count=count + 1, + ) + widths = (edges[1:] - edges[:-1]).astype(np.uint64) + indices = np.arange(count, dtype=np.uint64) + + # SplitMix64-style deterministic mixing. uint64 overflow is intentional. + with np.errstate(over="ignore"): + mixed = indices + np.uint64(seed) + mixed = (mixed ^ (mixed >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9) + mixed = (mixed ^ (mixed >> np.uint64(27))) * np.uint64(0x94D049BB133111EB) + mixed = mixed ^ (mixed >> np.uint64(31)) + + offsets = (mixed % widths).astype(np.int64) + positions = edges[:-1] + offsets + + # Keep full-range coverage as an explicit invariant while jittering the + # interior strata. This is useful for ordered/time-series diagnostics too. + positions[0] = 0 + positions[-1] = rows - 1 + return positions + + +def derive_streaming_profile_column_limit( + rows: int, + columns: int, + *, + mode: str = "standard", +) -> int: + """Return the deterministic full-stream column budget for a source shape. + + Ordinary datasets keep every column. Ultra-wide/high-cell-count sources are + projected before the full streaming profile pass so total scanned cells stay + bounded. The projection itself is selected by the caller from the source + schema; this function only resolves the allowed width. + """ + if mode not in _VALID_MODES: + raise ValueError(f"Unknown analysis mode: {mode}") + if rows < 0 or columns < 0: + raise ValueError("rows and columns must be non-negative.") + if columns == 0: + return 0 + if rows == 0: + return int(columns) + + cells = int(rows) * int(columns) + cell_budget = int(_STREAMING_PROFILE_CELL_BUDGETS[mode]) + if cells <= cell_budget and columns < 10_000: + return int(columns) + + by_cells = max(1, cell_budget // max(int(rows), 1)) + return max( + 1, + min( + int(columns), + int(by_cells), + int(_STREAMING_PROFILE_COLUMN_CAPS[mode]), + ), + ) + + +def derive_execution_budget( + rows: int, + columns: int, + *, + mode: str = "standard", +) -> ExecutionBudget: + """Derive a conservative execution policy from shape and analysis mode. + + The thresholds are deliberately simple and deterministic for now. They are + a compatibility layer for the future cost-based planner, where RAM, storage + metadata, native throughput, GPU availability, and user accuracy budgets can + refine the same object without changing analysis APIs. + """ + if mode not in _VALID_MODES: + raise ValueError(f"Unknown analysis mode: {mode}") + if rows < 0 or columns < 0: + raise ValueError("rows and columns must be non-negative.") + + cells = int(rows) * int(columns) + large_dataset = rows >= 100_000 or cells >= 10_000_000 + wide_dataset = columns >= 1_000 + ultra_wide_dataset = columns >= 10_000 + + if rows >= 100_000_000 or columns >= 100_000 or cells >= 1_000_000_000_000: + scale_class = "extreme" + elif rows >= 10_000_000 or columns >= 10_000 or cells >= 10_000_000_000: + scale_class = "very_large" + elif large_dataset or wide_dataset: + scale_class = "large" + else: + scale_class = "normal" + + presets = { + "quick": { + "quality": 1_000, + "deep": 2_000, + "bootstrap": 1_000, + "distribution": 2_000, + "pair": 2_000, + "anomaly": 5_000, + "time_series": 5_000, + "relationships": 10, + }, + "standard": { + "quality": 5_000, + "deep": 5_000, + "bootstrap": 2_500, + "distribution": 5_000, + "pair": 5_000, + "anomaly": 10_000, + "time_series": 10_000, + "relationships": 20, + }, + "deep": { + "quality": 10_000, + "deep": 10_000, + "bootstrap": 5_000, + "distribution": 10_000, + "pair": 10_000, + "anomaly": 25_000, + "time_series": 25_000, + "relationships": 50, + }, + "research": { + "quality": 20_000, + "deep": 20_000, + "bootstrap": 10_000, + "distribution": 20_000, + "pair": 20_000, + "anomaly": 50_000, + "time_series": 50_000, + "relationships": 100, + }, + } + selected = presets[mode] + + # Ultra-wide data must spend relationship budget more carefully. The future + # sparse feature-graph engine will replace this fixed cap with candidate + # generation rather than dense pair enumeration. + relationship_budget = int(selected["relationships"]) + if ultra_wide_dataset: + relationship_budget = min(relationship_budget, 10) + elif wide_dataset: + relationship_budget = min(relationship_budget, 20) + + # Memory-heavy modules should not be launched four-at-a-time simply because + # a machine exposes four Python workers. Large inputs default to sequential + # heavy execution until the scheduler gains RAM-aware token accounting. + heavy_parallelism = 1 if large_dataset or wide_dataset else 2 + + return ExecutionBudget( + mode=mode, + rows=int(rows), + columns=int(columns), + cells=cells, + scale_class=scale_class, + large_dataset=large_dataset, + wide_dataset=wide_dataset, + ultra_wide_dataset=ultra_wide_dataset, + quality_sample_rows=_bounded(selected["quality"], rows), + deep_statistics_sample_rows=_bounded(selected["deep"], rows), + bootstrap_sample_rows=_bounded(selected["bootstrap"], rows), + distribution_sample_rows=_bounded(selected["distribution"], rows), + pair_sample_rows=_bounded(selected["pair"], rows), + anomaly_sample_rows=_bounded(selected["anomaly"], rows), + time_series_sample_rows=_bounded(selected["time_series"], rows), + relationship_pair_budget=relationship_budget, + max_memory_heavy_parallelism=heavy_parallelism, + ) + + +def deterministic_sample_frame( + dataframe: pd.DataFrame, + max_rows: int, + *, + preserve_order: bool = False, +) -> tuple[pd.DataFrame, dict[str, Any]]: + """Return a deterministic bounded view and transparent sampling metadata.""" + source_rows = int(len(dataframe)) + if max_rows < 1: + raise ValueError("max_rows must be at least 1.") + + if source_rows <= max_rows: + return dataframe, { + "sampled": False, + "source_rows": source_rows, + "sample_rows": source_rows, + "strategy": "full", + } + + positions = _deterministic_stratified_positions(source_rows, max_rows) + sampled = dataframe.iloc[positions] + if not preserve_order: + # Positions stay sorted so time-aware callers can preserve ordering, while + # statistical callers receive a copy that is safe to mutate downstream. + sampled = sampled.copy() + + return sampled, { + "sampled": True, + "source_rows": source_rows, + "sample_rows": int(len(sampled)), + "strategy": "deterministic_stratified_jitter", + "preserve_order": bool(preserve_order), + "seed": int(_SAMPLE_SEED), + } diff --git a/src/framevitals/explainability.py b/src/framevitals/explainability.py index a66a081..ac3e4ae 100644 --- a/src/framevitals/explainability.py +++ b/src/framevitals/explainability.py @@ -9,7 +9,7 @@ For linear models, use shap.LinearExplainer. Fallback: sklearn permutation importance (model-agnostic). 3. Collapse one-hot expansions back to original feature names. - 4. Save a beeswarm summary plot to outputs/charts. + 4. Save a beeswarm summary plot when plotting support is installed. Public entry point: explain_winner(df, target_column, leaderboard_result) -> dict @@ -19,15 +19,9 @@ import warnings from pathlib import Path -from typing import Any -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt import numpy as np import pandas as pd - from sklearn.dummy import DummyClassifier, DummyRegressor from sklearn.ensemble import ( GradientBoostingClassifier, @@ -50,29 +44,38 @@ CHART_DIR = Path("static/charts") + # --------------------------------------------------------------------------- # Estimator rebuilds (mirrors model_leaderboard registry) # --------------------------------------------------------------------------- - def _rebuild_estimator(name: str, task_type: str): """Recreate the estimator by name. Returns None if unknown/missing.""" - name_lower = name.lower() - if task_type == "classification": if name == "DummyClassifier": return DummyClassifier(strategy="most_frequent") if name == "LogisticRegression": return LogisticRegression( - max_iter=2000, n_jobs=-1, class_weight="balanced", random_state=42 + max_iter=2000, + n_jobs=-1, + class_weight="balanced", + random_state=42, ) if name == "KNeighborsClassifier": return KNeighborsClassifier(n_neighbors=7) if name == "RandomForestClassifier": return RandomForestClassifier( - n_estimators=200, max_depth=10, random_state=42, n_jobs=-1, class_weight="balanced" + n_estimators=200, + max_depth=10, + random_state=42, + n_jobs=-1, + class_weight="balanced", ) if name == "GradientBoostingClassifier": - return GradientBoostingClassifier(n_estimators=150, max_depth=4, random_state=42) + return GradientBoostingClassifier( + n_estimators=150, + max_depth=4, + random_state=42, + ) if name == "XGBClassifier": try: from xgboost import XGBClassifier @@ -113,9 +116,18 @@ def _rebuild_estimator(name: str, task_type: str): if name == "KNeighborsRegressor": return KNeighborsRegressor(n_neighbors=7) if name == "RandomForestRegressor": - return RandomForestRegressor(n_estimators=200, max_depth=10, random_state=42, n_jobs=-1) + return RandomForestRegressor( + n_estimators=200, + max_depth=10, + random_state=42, + n_jobs=-1, + ) if name == "GradientBoostingRegressor": - return GradientBoostingRegressor(n_estimators=150, max_depth=4, random_state=42) + return GradientBoostingRegressor( + n_estimators=150, + max_depth=4, + random_state=42, + ) if name == "XGBRegressor": try: from xgboost import XGBRegressor @@ -168,7 +180,6 @@ def _is_linear_model(name: str) -> bool: # --------------------------------------------------------------------------- # One-hot collapse # --------------------------------------------------------------------------- - def _collapse_to_original( feature_names: list[str], importances: np.ndarray, @@ -176,13 +187,14 @@ def _collapse_to_original( categorical_features: list[str], ) -> list[dict]: """Sum one-hot expansions back to their original column names.""" - collapsed: dict[str, float] = {col: 0.0 for col in numeric_features + categorical_features} + collapsed: dict[str, float] = { + col: 0.0 for col in numeric_features + categorical_features + } for fname, value in zip(feature_names, importances): if fname in collapsed: collapsed[fname] += float(value) continue - # OneHot pattern: "_" matched = False for col in categorical_features: if fname.startswith(col + "_"): @@ -193,27 +205,34 @@ def _collapse_to_original( collapsed[fname] = float(value) rows = [ - {"feature": k, "importance": round(v, 6)} for k, v in collapsed.items() + {"feature": key, "importance": round(value, 6)} + for key, value in collapsed.items() ] - rows.sort(key=lambda r: abs(r["importance"]), reverse=True) + rows.sort(key=lambda row: abs(row["importance"]), reverse=True) return rows # --------------------------------------------------------------------------- -# SHAP plots +# Optional SHAP plot # --------------------------------------------------------------------------- - def _save_summary_plot( - shap_values, X_transformed, feature_names, dataset_id: str + shap_values, + X_transformed, + feature_names, + dataset_id: str, ) -> str | None: + """Write a SHAP summary plot when optional plotting support is available.""" try: - import shap + import matplotlib - CHART_DIR.mkdir( - parents=True, - exist_ok=True, - ) + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import shap + except Exception: + return None + try: + CHART_DIR.mkdir(parents=True, exist_ok=True) plt.figure(figsize=(9, 6)) shap.summary_plot( shap_values, @@ -225,17 +244,16 @@ def _save_summary_plot( path = CHART_DIR / f"{dataset_id}_shap_summary.png" plt.tight_layout() plt.savefig(path, dpi=150, bbox_inches="tight") - plt.close() return str(path) except Exception: - plt.close("all") return None + finally: + plt.close("all") # --------------------------------------------------------------------------- # Permutation importance fallback # --------------------------------------------------------------------------- - def _permutation_importance_block( pipeline: Pipeline, X: pd.DataFrame, @@ -247,17 +265,23 @@ def _permutation_importance_block( with warnings.catch_warnings(): warnings.simplefilter("ignore") result = permutation_importance( - pipeline, X, y, n_repeats=5, random_state=42, n_jobs=-1, scoring=None + pipeline, + X, + y, + n_repeats=5, + random_state=42, + n_jobs=-1, + scoring=None, ) rows = [ { "feature": col, - "importance": round(float(result.importances_mean[i]), 6), - "std": round(float(result.importances_std[i]), 6), + "importance": round(float(result.importances_mean[index]), 6), + "std": round(float(result.importances_std[index]), 6), } - for i, col in enumerate(X.columns) + for index, col in enumerate(X.columns) ] - rows.sort(key=lambda r: abs(r["importance"]), reverse=True) + rows.sort(key=lambda row: abs(row["importance"]), reverse=True) return {"available": True, "method": "permutation", "top_features": rows[:15]} except Exception as exc: return {"available": False, "reason": str(exc)} @@ -266,7 +290,6 @@ def _permutation_importance_block( # --------------------------------------------------------------------------- # Public entry point # --------------------------------------------------------------------------- - def explain_winner( df: pd.DataFrame, target_column: str, @@ -277,16 +300,8 @@ def explain_winner( """ Generate SHAP-based global + per-row explanations for the winning model. - Args: - df: Original dataframe. - target_column: Target column name. - leaderboard_result: Output of run_model_leaderboard. - dataset_id: Used for the SHAP summary chart filename. - sample_size: Number of test rows to compute SHAP on (cap for speed). - - Returns: - JSON-safe dict with method, global ranking, sample per-row stories, - and the path to the SHAP summary chart (if produced). + Plot generation is optional. The structured explanation and permutation + fallback remain usable without Matplotlib/Seaborn. """ if not leaderboard_result.get("available") or not leaderboard_result.get("winner"): return {"available": False, "message": "No leaderboard winner to explain."} @@ -296,11 +311,17 @@ def explain_winner( estimator = _rebuild_estimator(winner_name, task_type) if estimator is None: - return {"available": False, "message": f"Could not rebuild estimator: {winner_name}"} + return { + "available": False, + "message": f"Could not rebuild estimator: {winner_name}", + } prep = prepare_ml_matrix(df, target=target_column) if not prep["usable"]: - return {"available": False, "message": "Preprocessing produced no usable features."} + return { + "available": False, + "message": "Preprocessing produced no usable features.", + } X = prep["X"] y = prep["y"] @@ -310,21 +331,39 @@ def explain_winner( numeric_features = prep["numeric_features"] categorical_features = prep["categorical_features"] - preprocessor = build_sklearn_preprocessor(numeric_features, categorical_features) + preprocessor = build_sklearn_preprocessor( + numeric_features, + categorical_features, + ) pipeline = Pipeline([("pre", preprocessor), ("model", estimator)]) - # Fit once on a 75/25 split so we have a held-out slice for SHAP try: + stratify = ( + y + if ( + task_type == "classification" + and y.nunique() > 1 + and y.value_counts().min() >= 2 + ) + else None + ) X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.25, random_state=42, - stratify=y if task_type == "classification" and y.nunique() > 1 and y.value_counts().min() >= 2 else None, + X, + y, + test_size=0.25, + random_state=42, + stratify=stratify, ) except Exception: - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42) + X_train, X_test, y_train, y_test = train_test_split( + X, + y, + test_size=0.25, + random_state=42, + ) pipeline.fit(X_train, y_train) - # Sample the test set for SHAP speed if len(X_test) > sample_size: X_test_sample = X_test.sample(sample_size, random_state=42) else: @@ -333,7 +372,11 @@ def explain_winner( fitted_pre = pipeline.named_steps["pre"] fitted_model = pipeline.named_steps["model"] X_test_transformed = fitted_pre.transform(X_test_sample) - feature_names = get_transformed_feature_names(fitted_pre, numeric_features, categorical_features) + feature_names = get_transformed_feature_names( + fitted_pre, + numeric_features, + categorical_features, + ) method = None summary_chart_path: str | None = None @@ -341,7 +384,6 @@ def explain_winner( per_row_stories: list[dict] = [] error_messages: list[str] = [] - # Try SHAP try: import shap @@ -353,25 +395,27 @@ def explain_winner( shap_values_raw = explainer.shap_values(X_test_transformed) method = "shap.TreeExplainer" elif _is_linear_model(winner_name): - # LinearExplainer needs a background dataset - bg = fitted_pre.transform(X_train.sample(min(100, len(X_train)), random_state=42)) - explainer = shap.LinearExplainer(fitted_model, bg) + background = fitted_pre.transform( + X_train.sample(min(100, len(X_train)), random_state=42) + ) + explainer = shap.LinearExplainer(fitted_model, background) shap_values_raw = explainer.shap_values(X_test_transformed) method = "shap.LinearExplainer" else: - explainer = None shap_values_raw = None if shap_values_raw is not None: - # For multiclass tree explainers, shap returns a list per class -> use mean abs across classes if isinstance(shap_values_raw, list): - shap_values = np.mean([np.abs(arr) for arr in shap_values_raw], axis=0) - # Signed values for plotting: take positive class or mean across classes + shap_values = np.mean( + [np.abs(array) for array in shap_values_raw], + axis=0, + ) shap_values_for_plot = ( - shap_values_raw[1] if len(shap_values_raw) == 2 else shap_values_raw[0] + shap_values_raw[1] + if len(shap_values_raw) == 2 + else shap_values_raw[0] ) elif shap_values_raw.ndim == 3: - # newer SHAP returns (n_samples, n_features, n_classes) shap_values = np.mean(np.abs(shap_values_raw), axis=2) shap_values_for_plot = shap_values_raw[:, :, 0] else: @@ -380,38 +424,57 @@ def explain_winner( mean_abs = shap_values.mean(axis=0) global_rows = _collapse_to_original( - feature_names, mean_abs, numeric_features, categorical_features + feature_names, + mean_abs, + numeric_features, + categorical_features, ) - # Per-row stories: top-3 rows by total |shap| row_totals = shap_values.sum(axis=1) top_idx = np.argsort(-row_totals)[:3] - for i, ridx in enumerate(top_idx): - contributions = list(zip(feature_names, shap_values_for_plot[ridx])) - contributions.sort(key=lambda t: abs(t[1]), reverse=True) + for rank, row_index in enumerate(top_idx, start=1): + contributions = list( + zip(feature_names, shap_values_for_plot[row_index]) + ) + contributions.sort(key=lambda item: abs(item[1]), reverse=True) story = [ - {"feature": fname, "shap_value": round(float(val), 6)} - for fname, val in contributions[:5] + { + "feature": feature, + "shap_value": round(float(value), 6), + } + for feature, value in contributions[:5] ] + source_index = X_test_sample.index[row_index] per_row_stories.append({ - "rank": i + 1, - "row_index": int(X_test_sample.index[ridx]) if hasattr(X_test_sample.index[ridx], "__int__") else str(X_test_sample.index[ridx]), + "rank": rank, + "row_index": ( + int(source_index) + if hasattr(source_index, "__int__") + else str(source_index) + ), "top_contributions": story, }) summary_chart_path = _save_summary_plot( - shap_values_for_plot, X_test_transformed, feature_names, dataset_id + shap_values_for_plot, + X_test_transformed, + feature_names, + dataset_id, ) except Exception as exc: error_messages.append(f"SHAP failed: {exc}") method = None - # Permutation importance — always run as cross-check / fallback - perm = _permutation_importance_block(pipeline, X_test, y_test, numeric_features, categorical_features) + perm = _permutation_importance_block( + pipeline, + X_test, + y_test, + numeric_features, + categorical_features, + ) if not global_rows and perm.get("available"): - # Use permutation as the global ranking global_rows = perm["top_features"] method = method or "permutation_importance" diff --git a/src/framevitals/fast_anomaly.py b/src/framevitals/fast_anomaly.py new file mode 100644 index 0000000..efa0896 --- /dev/null +++ b/src/framevitals/fast_anomaly.py @@ -0,0 +1,277 @@ +"""Fast bounded anomaly screening for standard and deep analysis modes. + +The scanner combines robust per-feature deviation, sparse random-projection tail +scores, and projection-density rarity. It is fully vectorized and avoids tree, +nearest-neighbour, and covariance fitting. Research mode can still run the +heavier classical/neural ensemble for confirmation. +""" + +from __future__ import annotations + +import math +from typing import Any + +import numpy as np +import pandas as pd + + +def _to_unit(values: np.ndarray) -> np.ndarray: + values = np.asarray(values, dtype=np.float64) + finite = np.where(np.isfinite(values), values, np.nan) + if finite.size == 0 or np.all(np.isnan(finite)): + return np.zeros_like(values, dtype=np.float64) + lo = float(np.nanmin(finite)) + hi = float(np.nanmax(finite)) + if hi - lo <= 1e-12: + return np.zeros_like(values, dtype=np.float64) + out = (finite - lo) / (hi - lo) + return np.where(np.isfinite(out), out, 0.0) + + +def _summary(scores: np.ndarray) -> dict[str, float]: + scores = np.asarray(scores, dtype=np.float64) + return { + "mean": round(float(np.mean(scores)), 4), + "median": round(float(np.median(scores)), 4), + "p95": round(float(np.quantile(scores, 0.95)), 4), + "p99": round(float(np.quantile(scores, 0.99)), 4), + "max": round(float(np.max(scores)), 4), + } + + +def _prepare_numeric( + dataframe: pd.DataFrame, + *, + max_columns: int, +) -> tuple[np.ndarray | None, list[str], dict[str, Any]]: + numeric = dataframe.select_dtypes(include=[np.number]).replace( + [np.inf, -np.inf], + np.nan, + ) + original_columns = list(numeric.columns) + dropped_all_missing: list[str] = [] + dropped_constant: list[str] = [] + + for column in list(numeric.columns): + valid = numeric[column].dropna() + if valid.empty: + numeric = numeric.drop(columns=[column]) + dropped_all_missing.append(str(column)) + elif valid.nunique() <= 1: + numeric = numeric.drop(columns=[column]) + dropped_constant.append(str(column)) + + if numeric.empty: + return None, [], { + "numeric_columns_found": len(original_columns), + "used_columns": [], + "dropped_all_missing_columns": dropped_all_missing, + "dropped_constant_columns": dropped_constant, + "truncated_columns": False, + } + + # Prefer columns with usable coverage and variation. This keeps ID-like or + # mostly-empty columns from crowding out informative dimensions. + variance = numeric.var(axis=0, skipna=True).fillna(0.0) + coverage = numeric.notna().mean(axis=0) + rank = (np.log1p(variance.abs()) + coverage).sort_values(ascending=False) + selected = rank.head(max_columns).index.tolist() + numeric = numeric[selected] + + medians = numeric.median(axis=0, skipna=True) + numeric = numeric.fillna(medians).fillna(0.0) + matrix = numeric.to_numpy(dtype=np.float64) + + center = np.median(matrix, axis=0) + mad = np.median(np.abs(matrix - center), axis=0) + std = np.std(matrix, axis=0, ddof=1) + scale = 1.4826 * mad + scale = np.where(scale > 1e-12, scale, np.where(std > 1e-12, std, 1.0)) + standardized = (matrix - center) / scale + + return standardized, [str(column) for column in selected], { + "numeric_columns_found": len(original_columns), + "used_columns": [str(column) for column in selected], + "dropped_all_missing_columns": dropped_all_missing, + "dropped_constant_columns": dropped_constant, + "truncated_columns": len(selected) < len(original_columns) - len(dropped_all_missing) - len(dropped_constant), + "scaling": "median_mad_with_std_fallback", + } + + +def _projection_density_scores(projected: np.ndarray, bins: int = 32) -> np.ndarray: + n_rows, n_projections = projected.shape + if n_rows == 0 or n_projections == 0: + return np.zeros(n_rows, dtype=np.float64) + rarity = np.zeros((n_rows, n_projections), dtype=np.float64) + + for index in range(n_projections): + values = projected[:, index] + counts, edges = np.histogram(values, bins=min(bins, max(4, int(np.sqrt(n_rows))))) + positions = np.searchsorted(edges, values, side="right") - 1 + positions = np.clip(positions, 0, len(counts) - 1) + row_counts = counts[positions] + rarity[:, index] = -np.log((row_counts + 1.0) / (n_rows + len(counts))) + + return _to_unit(np.mean(rarity, axis=1)) + + +def fast_anomaly_scan( + dataframe: pd.DataFrame, + *, + contamination: float = 0.05, + threshold: float = 0.6, + max_columns: int = 24, + projections: int = 12, + top_k: int = 25, + random_state: int = 42, +) -> dict[str, Any]: + """Return a fast explainable anomaly screen over a bounded dataframe.""" + if max_columns < 1: + raise ValueError("max_columns must be at least 1.") + if projections < 2: + raise ValueError("projections must be at least 2.") + if top_k < 1: + raise ValueError("top_k must be at least 1.") + if not 0.0 <= threshold <= 1.0: + raise ValueError("threshold must be between 0 and 1.") + contamination = float(np.clip(contamination, 0.001, 0.5)) + + matrix, used_columns, preparation = _prepare_numeric( + dataframe, + max_columns=max_columns, + ) + if matrix is None or len(matrix) < 20: + return { + "available": False, + "message": "Need at least 20 rows and one non-constant numeric column.", + "preparation": preparation, + } + + n_rows, n_features = matrix.shape + robust_abs = np.abs(matrix) + top_feature_count = min(3, n_features) + if top_feature_count == n_features: + robust_raw = robust_abs.mean(axis=1) + else: + top = np.partition(robust_abs, n_features - top_feature_count, axis=1)[ + :, -top_feature_count: + ] + robust_raw = top.mean(axis=1) + robust_score = _to_unit(robust_raw) + + rng = np.random.default_rng(random_state) + projection_matrix = rng.choice( + np.array([-1.0, 0.0, 1.0]), + size=(n_features, projections), + p=[0.25, 0.50, 0.25], + ) + for index in range(projections): + nonzero = np.count_nonzero(projection_matrix[:, index]) + if nonzero == 0: + projection_matrix[index % n_features, index] = 1.0 + nonzero = 1 + projection_matrix[:, index] /= math.sqrt(nonzero) + + projected = matrix @ projection_matrix + projection_center = np.median(projected, axis=0) + projection_mad = np.median(np.abs(projected - projection_center), axis=0) + projection_scale = np.where(1.4826 * projection_mad > 1e-12, 1.4826 * projection_mad, 1.0) + projection_z = np.abs((projected - projection_center) / projection_scale) + projection_top = np.partition( + projection_z, + max(0, projections - min(3, projections)), + axis=1, + )[:, -min(3, projections):] + projection_tail_score = _to_unit(projection_top.mean(axis=1)) + density_score = _projection_density_scores(projected) + + detector_scores = { + "robust_feature_deviation": robust_score, + "random_projection_tail": projection_tail_score, + "random_projection_density": density_score, + } + ensemble = ( + 0.50 * robust_score + + 0.35 * projection_tail_score + + 0.15 * density_score + ) + ensemble = _to_unit(ensemble) + + detector_names = list(detector_scores) + vote_thresholds = { + name: float(np.quantile(scores, 1.0 - contamination)) + for name, scores in detector_scores.items() + } + votes = np.column_stack([ + detector_scores[name] >= vote_thresholds[name] + for name in detector_names + ]) + agreement_count = votes.sum(axis=1) + majority_required = max(1, math.ceil(len(detector_names) / 2)) + consensus_mask = agreement_count >= majority_required + flagged_mask = ensemble >= threshold + + top_indices = np.argsort(ensemble)[::-1][: min(top_k, n_rows)] + top_rows: list[dict[str, Any]] = [] + for position in top_indices: + feature_order = np.argsort(robust_abs[position])[::-1][: min(3, n_features)] + top_rows.append({ + "row_index": ( + int(dataframe.index[position]) + if isinstance(dataframe.index[position], (int, np.integer)) + else str(dataframe.index[position]) + ), + "robust_feature_deviation": round(float(robust_score[position]), 4), + "random_projection_tail": round(float(projection_tail_score[position]), 4), + "random_projection_density": round(float(density_score[position]), 4), + "ensemble": round(float(ensemble[position]), 4), + "flagged": bool(flagged_mask[position]), + "agreement_count": int(agreement_count[position]), + "agreement_fraction": round(float(agreement_count[position] / len(detector_names)), 4), + "top_feature_deviations": [ + { + "feature": used_columns[int(feature_index)], + "standardized_deviation": round(float(robust_abs[position, feature_index]), 4), + "direction": "high" if matrix[position, feature_index] >= 0 else "low", + } + for feature_index in feature_order + ], + }) + + flagged_count = int(flagged_mask.sum()) + consensus_count = int(consensus_mask.sum()) + return { + "available": True, + "method": "fast_robust_random_projection", + "n_rows_scored": int(n_rows), + "used_columns": used_columns, + "preparation": preparation, + "detectors_run": detector_names, + "detectors_failed": {}, + "detectors_skipped": {}, + "detector_summaries": { + name: _summary(scores) for name, scores in detector_scores.items() + }, + "detector_vote_thresholds": { + name: round(value, 4) for name, value in vote_thresholds.items() + }, + "threshold": float(threshold), + "contamination": contamination, + "expected_anomaly_count": int(math.ceil(n_rows * contamination)), + "flagged_count": flagged_count, + "flagged_fraction": round(float(flagged_count / n_rows), 4), + "consensus": { + "majority_detectors_required": majority_required, + "flagged_count": consensus_count, + "flagged_fraction": round(float(consensus_count / n_rows), 4), + }, + "ensemble_summary": _summary(ensemble), + "top_rows": top_rows, + "projection_count": int(projections), + "interpretation": ( + "Standard/deep anomaly screening combines robust feature deviations, sparse " + "random-projection tail behaviour, and projection-density rarity. Research mode " + "can confirm candidates with the heavier classical and neural ensembles." + ), + } diff --git a/src/framevitals/fast_deep_statistics.py b/src/framevitals/fast_deep_statistics.py new file mode 100644 index 0000000..564c28e --- /dev/null +++ b/src/framevitals/fast_deep_statistics.py @@ -0,0 +1,281 @@ +"""Fast deep-statistics execution for non-research analysis modes. + +The public ``deep_statistics_v2`` implementation keeps its BCa bootstrap +semantics. Budgeted quick/standard/deep execution can use this module to avoid +thousands of resamples per numeric column while preserving the rest of the v2 +diagnostic battery. + +Mean confidence intervals use the exact Student-t construction and median +confidence intervals use distribution-free order statistics derived from the +Binomial(n, 0.5) model. Both are deterministic and O(n). +""" + +from __future__ import annotations + +import math +from typing import Any + +import numpy as np +import pandas as pd +from scipy import stats + +from framevitals.deep_statistics_v2 import ( + _classify_kurtosis, + _classify_skew, + _fit_best_distribution, + _group_difference_test, + _normality, + _numeric_pair_stats, + _outlier_flags, + _point_biserial, + _safe_float, + run_deep_statistics_v2, +) + + +def _finite_numeric(series: pd.Series) -> np.ndarray: + values = pd.to_numeric(series, errors="coerce").to_numpy( + dtype="float64", + na_value=np.nan, + ) + return values[np.isfinite(values)] + + +def fast_mean_ci( + series: pd.Series, + confidence: float = 0.95, +) -> dict[str, Any]: + """Student-t confidence interval for the arithmetic mean in O(n).""" + values = _finite_numeric(series) + n = int(values.size) + if n < 20: + return {"available": False, "reason": "n<20"} + + mean = float(values.mean()) + std = float(values.std(ddof=1)) + if not math.isfinite(std): + return {"available": False, "reason": "non-finite sample variance"} + if std <= 1.0e-15: + return { + "available": True, + "low": _safe_float(mean), + "high": _safe_float(mean), + "method": "student_t", + "n_resamples": 0, + } + + alpha = 1.0 - float(confidence) + critical = float(stats.t.ppf(1.0 - alpha / 2.0, n - 1)) + half_width = critical * std / math.sqrt(n) + return { + "available": True, + "low": _safe_float(mean - half_width), + "high": _safe_float(mean + half_width), + "method": "student_t", + "n_resamples": 0, + } + + +def fast_median_ci( + series: pd.Series, + confidence: float = 0.95, +) -> dict[str, Any]: + """Distribution-free median CI via two selected order statistics. + + ``np.partition`` selects only the two required ranks, so no full sort or + resampling matrix is allocated. + """ + values = _finite_numeric(series) + n = int(values.size) + if n < 20: + return {"available": False, "reason": "n<20"} + + alpha = 1.0 - float(confidence) + rank = int(stats.binom.ppf(alpha / 2.0, n, 0.5)) + rank = max(1, min(rank, n // 2)) + + low_index = rank - 1 + high_index = n - rank + partitioned = np.partition(values, (low_index, high_index)) + + return { + "available": True, + "low": _safe_float(partitioned[low_index]), + "high": _safe_float(partitioned[high_index]), + "method": "distribution_free_order_statistic", + "n_resamples": 0, + "rank_low": int(rank), + "rank_high": int(n - rank + 1), + } + + +def _numeric_column_stats_fast(series: pd.Series) -> dict[str, Any]: + s = series.dropna() + if s.empty: + return {"status": "empty"} + + skew = _safe_float(s.skew()) + kurt = _safe_float(s.kurtosis()) + q1, q3 = _safe_float(s.quantile(0.25)), _safe_float(s.quantile(0.75)) + iqr = (q3 - q1) if (q1 is not None and q3 is not None) else None + + return { + "count": int(s.count()), + "mean": _safe_float(s.mean()), + "median": _safe_float(s.median()), + "std": _safe_float(s.std()), + "min": _safe_float(s.min()), + "max": _safe_float(s.max()), + "q1": q1, + "q3": q3, + "iqr": _safe_float(iqr), + "skewness": skew, + "skewness_label": _classify_skew(skew), + "kurtosis": kurt, + "kurtosis_label": _classify_kurtosis(kurt), + "outliers": _outlier_flags(series), + "normality": _normality(series), + "distribution_fit": _fit_best_distribution(series), + "bootstrap_mean_ci": fast_mean_ci(series), + "bootstrap_median_ci": fast_median_ci(series), + } + + +def _numeric_bivariate( + df: pd.DataFrame, + numeric_cols: list[str], + categorical_cols: list[str], + *, + max_pairs: int, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: + numeric_pairs: list[dict[str, Any]] = [] + budget = max_pairs + for index, left in enumerate(numeric_cols): + for right in numeric_cols[index + 1 :]: + if budget <= 0: + break + payload = _numeric_pair_stats(df[left], df[right]) + if payload.get("available"): + numeric_pairs.append( + {"column_a": left, "column_b": right, **payload} + ) + budget -= 1 + if budget <= 0: + break + numeric_pairs.sort( + key=lambda item: abs(item.get("pearson", {}).get("r") or 0), + reverse=True, + ) + + binary_numeric_pairs: list[dict[str, Any]] = [] + budget = max_pairs + for category in categorical_cols: + if df[category].nunique(dropna=True) != 2: + continue + for numeric in numeric_cols: + if budget <= 0: + break + payload = _point_biserial(df[category], df[numeric]) + if payload.get("available"): + binary_numeric_pairs.append( + { + "binary_column": category, + "numeric_column": numeric, + **payload, + } + ) + budget -= 1 + if budget <= 0: + break + + group_difference_tests: list[dict[str, Any]] = [] + budget = max_pairs + for category in categorical_cols: + unique = df[category].nunique(dropna=True) + if unique < 2 or unique > 6: + continue + for numeric in numeric_cols: + if budget <= 0: + break + payload = _group_difference_test(df[numeric], df[category]) + if payload.get("available"): + group_difference_tests.append( + { + "group_column": category, + "numeric_column": numeric, + **payload, + } + ) + budget -= 1 + if budget <= 0: + break + + return numeric_pairs, binary_numeric_pairs, group_difference_tests + + +def _categorical_columns(df: pd.DataFrame) -> list[str]: + """Select non-numeric categorical/string columns without dtype migration warnings.""" + columns: list[str] = [] + for column in df.columns: + dtype = df[column].dtype + if ( + pd.api.types.is_object_dtype(dtype) + or pd.api.types.is_string_dtype(dtype) + or isinstance(dtype, pd.CategoricalDtype) + or pd.api.types.is_bool_dtype(dtype) + ): + columns.append(str(column)) + return columns + + +def run_fast_deep_statistics_v2( + df: pd.DataFrame, + max_pairs: int = 20, +) -> dict[str, Any]: + """Run the v2 diagnostic battery with O(n) confidence intervals.""" + numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist() + categorical_cols = _categorical_columns(df) + + categorical_view = ( + df.loc[:, categorical_cols] + if categorical_cols + else pd.DataFrame(index=df.index) + ) + result = run_deep_statistics_v2(categorical_view, max_pairs=max_pairs) + + result["numeric_columns"] = numeric_cols + result["numeric_statistics"] = { + column: _numeric_column_stats_fast(df[column]) + for column in numeric_cols + } + + ( + numeric_pairs, + binary_numeric_pairs, + group_difference_tests, + ) = _numeric_bivariate( + df, + numeric_cols, + categorical_cols, + max_pairs=max_pairs, + ) + + result["bivariate"]["numeric_pairs"] = numeric_pairs + result["bivariate"]["binary_numeric_pairs"] = binary_numeric_pairs + result["bivariate"]["group_difference_tests"] = group_difference_tests + + result["summary"].update( + { + "numeric_count": len(numeric_cols), + "numeric_pairs_tested": len(numeric_pairs), + "binary_numeric_pairs_tested": len(binary_numeric_pairs), + "group_difference_tests_run": len(group_difference_tests), + } + ) + result["inference"] = { + "method": "fast_closed_form_and_order_statistics", + "mean_ci": "student_t", + "median_ci": "distribution_free_order_statistic", + "bootstrap_resamples": 0, + } + return result diff --git a/src/framevitals/findings.py b/src/framevitals/findings.py new file mode 100644 index 0000000..f484841 --- /dev/null +++ b/src/framevitals/findings.py @@ -0,0 +1,361 @@ +"""Normalized findings for FrameVitals analysis results. + +FrameVitals diagnostics originate in several deterministic analysis layers. This +module translates their already-computed warnings/signals into one stable, +machine-readable finding shape without changing diagnostic thresholds. +""" + +from __future__ import annotations + +import re +from typing import Any, Iterable, Mapping + + +_SEVERITY_MAP = { + "high": "high", + "critical": "critical", + "medium": "medium", + "moderate": "medium", + "low": "low", + "informational": "info", + "info": "info", + "none": "none", +} + +_SEVERITY_ORDER = { + "critical": 0, + "high": 1, + "medium": 2, + "low": 3, + "info": 4, + "none": 5, +} + + +def _slug(value: str) -> str: + normalized = re.sub(r"[^a-z0-9]+", "_", value.strip().lower()) + return normalized.strip("_") or "finding" + + +def normalize_severity(value: Any) -> str: + """Return a stable lower-case severity label.""" + if value is None: + return "none" + text = str(value).strip().lower() + return _SEVERITY_MAP.get(text, text or "none") + + +def _sort_findings(findings: list[dict[str, Any]]) -> list[dict[str, Any]]: + findings.sort( + key=lambda item: ( + _SEVERITY_ORDER.get(str(item.get("severity")), 99), + str(item.get("code", "")), + ) + ) + return findings + + +def _is_actionable_signal(signal: dict[str, Any]) -> bool: + status = str(signal.get("status", "")).strip().lower() + name = str(signal.get("name", "")).strip().lower() + + if status == "review": + return True + if name == "ml readiness" and status not in {"ready", "good"}: + return True + return False + + +def findings_from_signals(signals: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + """Normalize existing display signals into actionable findings.""" + findings: list[dict[str, Any]] = [] + + for signal in signals: + if not isinstance(signal, dict) or not _is_actionable_signal(signal): + continue + + name = str(signal.get("name") or "Finding") + findings.append({ + "code": f"signal.{_slug(name)}", + "title": name, + "severity": normalize_severity(signal.get("severity")), + "scope": "dataset", + "status": signal.get("status"), + "evidence": signal.get("evidence") or "", + "recommendation": signal.get("recommendation") or "", + "method": "signal_engine", + "confidence": "deterministic", + }) + + return _sort_findings(findings) + + +def findings_from_target_intelligence( + target_intelligence: Mapping[str, Any] | None, +) -> list[dict[str, Any]]: + """Normalize target-quality/leakage warnings into standard findings.""" + if not isinstance(target_intelligence, Mapping) or not target_intelligence.get("available"): + return [] + + target = str(target_intelligence.get("target_column") or "target") + warnings = target_intelligence.get("warnings", []) + if not isinstance(warnings, list): + return [] + + findings: list[dict[str, Any]] = [] + for warning in warnings: + if not isinstance(warning, Mapping): + continue + code = str(warning.get("code") or "target.warning") + evidence = str(warning.get("message") or "Target review is recommended.") + + if code == "target.id_like": + title = "Identifier-like target" + recommendation = ( + f"Verify that '{target}' represents an outcome rather than a record identifier " + "before training a supervised model." + ) + elif code == "target.high_missingness": + title = "High target missingness" + recommendation = ( + "Resolve or explicitly exclude rows with missing target labels before model training." + ) + elif code == "target.high_cardinality_classification": + title = "High-cardinality classification target" + recommendation = ( + "Confirm that the target classes are intentional and have enough examples for reliable evaluation." + ) + elif code.startswith("target.leakage."): + feature = code.removeprefix("target.leakage.") + title = f"Potential target leakage: {feature}" + recommendation = ( + f"Review or remove '{feature}' before training if it contains information that would " + "not be available at prediction time." + ) + else: + title = "Target review" + recommendation = "Review the selected target and its relationship to the input features." + + findings.append({ + "code": code, + "title": title, + "severity": normalize_severity(warning.get("severity")), + "scope": "target", + "status": "Review", + "evidence": evidence, + "recommendation": recommendation, + "method": "target_intelligence", + "confidence": "deterministic", + }) + + return _sort_findings(findings) + + +def _quality_finding( + *, + code: str, + title: str, + severity: Any, + evidence: str, + recommendation: str, + scope: str = "column", +) -> dict[str, Any]: + return { + "code": code, + "title": title, + "severity": normalize_severity(severity), + "scope": scope, + "status": "Review", + "evidence": evidence, + "recommendation": recommendation, + "method": "quality_diagnostics", + "confidence": "deterministic", + } + + +def findings_from_quality_diagnostics( + diagnostics: Mapping[str, Any] | None, +) -> list[dict[str, Any]]: + """Translate deterministic quality diagnostics into actionable findings.""" + if not isinstance(diagnostics, Mapping) or not diagnostics.get("available"): + return [] + + findings: list[dict[str, Any]] = [] + + for item in diagnostics.get("identifier_duplicates", []): + if not isinstance(item, Mapping): + continue + column = str(item.get("column") or "identifier") + duplicate_rows = int(item.get("duplicate_rows") or 0) + findings.append(_quality_finding( + code=f"quality.identifier_duplicates.{_slug(column)}", + title=f"Duplicate identifiers: {column}", + severity=item.get("severity", "high"), + evidence=f"{duplicate_rows} rows share duplicated values in identifier-like column '{column}'.", + recommendation=( + f"Verify uniqueness rules for '{column}' and resolve duplicated identifiers before joins, " + "deduplication, or model evaluation." + ), + )) + + for item in diagnostics.get("duplicate_columns", []): + if not isinstance(item, Mapping): + continue + canonical = str(item.get("canonical_column") or "column") + duplicates = [str(value) for value in item.get("duplicate_columns", [])] + if not duplicates: + continue + findings.append(_quality_finding( + code=f"quality.duplicate_columns.{_slug(canonical)}", + title=f"Duplicate columns: {canonical}", + severity=item.get("severity", "medium"), + evidence=( + f"'{canonical}' is exactly duplicated by: {', '.join(duplicates)}." + ), + recommendation=( + "Remove or explicitly document redundant columns to reduce ambiguity, memory usage, and leakage risk." + ), + )) + + for item in diagnostics.get("quasi_constant_columns", []): + if not isinstance(item, Mapping): + continue + column = str(item.get("column") or "column") + ratio = float(item.get("top_value_ratio") or 0) + findings.append(_quality_finding( + code=f"quality.quasi_constant.{_slug(column)}", + title=f"Quasi-constant column: {column}", + severity=item.get("severity", "low"), + evidence=f"One value represents approximately {ratio:.1%} of non-missing sampled values.", + recommendation=( + f"Review whether '{column}' carries useful signal; near-constant features often add little analytical value." + ), + )) + + for item in diagnostics.get("coercion_candidates", []): + if not isinstance(item, Mapping): + continue + column = str(item.get("column") or "column") + suggested = str(item.get("suggested_type") or "structured") + ratio = float(item.get("parse_ratio") or 0) + findings.append(_quality_finding( + code=f"quality.coercion.{_slug(column)}", + title=f"Type coercion candidate: {column}", + severity=item.get("severity", "low"), + evidence=f"Approximately {ratio:.1%} of sampled values parse cleanly as {suggested}.", + recommendation=( + f"Consider converting '{column}' to {suggested} after reviewing non-parsing values and preserving intended semantics." + ), + )) + + for item in diagnostics.get("category_normalisation", []): + if not isinstance(item, Mapping): + continue + column = str(item.get("column") or "column") + groups = int(item.get("variant_group_count") or 0) + findings.append(_quality_finding( + code=f"quality.category_normalisation.{_slug(column)}", + title=f"Category normalization issue: {column}", + severity=item.get("severity", "medium"), + evidence=f"Detected {groups} category groups that differ only by case or surrounding whitespace.", + recommendation=( + f"Normalize whitespace/casing in '{column}' with an explicit mapping before grouping, validation, or modelling." + ), + )) + + for item in diagnostics.get("blank_strings", []): + if not isinstance(item, Mapping): + continue + column = str(item.get("column") or "column") + count = int(item.get("blank_count_in_sample") or 0) + findings.append(_quality_finding( + code=f"quality.blank_strings.{_slug(column)}", + title=f"Blank strings: {column}", + severity=item.get("severity", "medium"), + evidence=f"Found {count} blank/whitespace-only values in the diagnostic sample.", + recommendation=( + f"Treat blank strings in '{column}' consistently as missing values or a documented category." + ), + )) + + for item in diagnostics.get("infinite_values", []): + if not isinstance(item, Mapping): + continue + column = str(item.get("column") or "column") + count = int(item.get("infinite_count_in_sample") or 0) + findings.append(_quality_finding( + code=f"quality.infinite_values.{_slug(column)}", + title=f"Infinite numeric values: {column}", + severity=item.get("severity", "high"), + evidence=f"Found {count} positive/negative infinite values in the diagnostic sample.", + recommendation=( + f"Replace or explicitly handle infinities in '{column}' before statistics, scaling, or model training." + ), + )) + + for item in diagnostics.get("mixed_object_types", []): + if not isinstance(item, Mapping): + continue + column = str(item.get("column") or "column") + types = item.get("python_types", {}) + findings.append(_quality_finding( + code=f"quality.mixed_object_types.{_slug(column)}", + title=f"Mixed Python types: {column}", + severity=item.get("severity", "medium"), + evidence=f"Object column contains multiple runtime value types: {types}.", + recommendation=( + f"Standardize the representation of '{column}' before serialization, joins, validation, or type conversion." + ), + )) + + for item in diagnostics.get("missingness_relationships", []): + if not isinstance(item, Mapping): + continue + columns = [str(value) for value in item.get("columns", [])] + if len(columns) != 2: + continue + jaccard = float(item.get("jaccard") or 0) + findings.append(_quality_finding( + code=f"quality.missingness_relationship.{_slug(columns[0])}.{_slug(columns[1])}", + title=f"Linked missingness: {columns[0]} + {columns[1]}", + severity=item.get("severity", "low"), + evidence=f"Their missing-value masks have Jaccard similarity {jaccard:.2f} in the diagnostic sample.", + recommendation=( + "Investigate whether these columns are jointly missing because of one upstream process, segment, or collection rule." + ), + scope="dataset", + )) + + return _sort_findings(findings) + + +def merge_findings(*groups: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge finding groups, de-duplicating by stable code.""" + merged: list[dict[str, Any]] = [] + seen: set[str] = set() + for group in groups: + for finding in group: + code = str(finding.get("code") or "") + if code and code in seen: + continue + if code: + seen.add(code) + merged.append(dict(finding)) + return _sort_findings(merged) + + +def recommendations_from_findings( + findings: Iterable[dict[str, Any]], +) -> list[str]: + """Return de-duplicated actionable recommendations in finding order.""" + recommendations: list[str] = [] + seen: set[str] = set() + + for finding in findings: + recommendation = str(finding.get("recommendation") or "").strip() + if not recommendation or recommendation in seen: + continue + seen.add(recommendation) + recommendations.append(recommendation) + + return recommendations diff --git a/src/framevitals/focused.py b/src/framevitals/focused.py new file mode 100644 index 0000000..20e76a5 --- /dev/null +++ b/src/framevitals/focused.py @@ -0,0 +1,532 @@ +"""Focused FrameVitals analysis entry points. + +This module intentionally does not import the full analysis pipeline. Public +calls such as ``fv.profile`` and ``fv.statistics`` execute only the work the +caller requested. Individual analysis implementations are imported lazily too, +so a profile-only call does not load sklearn/statsmodels/deep-stat modules. +""" + +from __future__ import annotations + +from typing import Any + +import pandas as pd + +from framevitals.provenance import ( + execution_provenance, + load_fully_materializes, + normalize_execution, +) +from framevitals.result import DiagnosticResult +from framevitals.sources import StreamingDatasetSource, resolve_source + + +DataInput = Any + + +def _load(data: DataInput, *, label: str = "Dataset") -> tuple[pd.DataFrame, str]: + try: + source = resolve_source(data) + except (TypeError, ValueError, FileNotFoundError) as exc: + if label == "Dataset": + raise + raise type(exc)(str(exc).replace("Dataset", label, 1)) from exc + + metadata = source.inspect() + dataframe = source.load() + return dataframe, metadata.name + + +def _named( + payload: dict[str, Any], + source_name: str, + *, + diagnostic: str, +) -> DiagnosticResult: + return DiagnosticResult( + {"dataset_name": source_name, **payload}, + diagnostic=diagnostic, + ) + + +def profile(data: DataInput) -> DiagnosticResult: + """Profile a dataset, streaming Arrow-capable file sources when available.""" + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.streaming_profile import build_streaming_profile + + return _named( + build_streaming_profile(source), + metadata.name, + diagnostic="profile", + ) + + from framevitals.profiler import build_profile + + dataframe = source.load() + return _named( + build_profile(dataframe), + metadata.name, + diagnostic="profile", + ) + + +def roles(data: DataInput) -> DiagnosticResult: + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.streaming_profile import build_streaming_profile + from framevitals.streaming_roles import infer_streaming_column_roles + + dataset_profile, sample = build_streaming_profile( + source, + sample_rows=5_000, + return_sample=True, + ) + payload = infer_streaming_column_roles(sample, profile=dataset_profile) + return _named(payload, metadata.name, diagnostic="roles") + + from framevitals.column_roles import infer_column_roles, summarize_roles + + dataframe = source.load() + column_roles = infer_column_roles(dataframe) + return _named( + { + "columns": column_roles, + "summary": summarize_roles(column_roles), + }, + metadata.name, + diagnostic="roles", + ) + + +def health(data: DataInput) -> DiagnosticResult: + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.health_score import calculate_health_score_from_profile_sample + from framevitals.streaming_profile import build_streaming_profile + + dataset_profile, sample = build_streaming_profile(source, return_sample=True) + payload = calculate_health_score_from_profile_sample(dataset_profile, sample) + return _named(payload, metadata.name, diagnostic="health") + + from framevitals.health_score import calculate_health_score + from framevitals.profiler import build_profile + + dataframe = source.load() + dataset_profile = build_profile(dataframe) + return _named( + calculate_health_score(dataframe, dataset_profile), + metadata.name, + diagnostic="health", + ) + + +def ml_readiness(data: DataInput) -> DiagnosticResult: + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.ml_readiness import calculate_ml_readiness_from_profile + from framevitals.streaming_profile import build_streaming_profile + + dataset_profile = build_streaming_profile(source) + return _named( + calculate_ml_readiness_from_profile(dataset_profile), + metadata.name, + diagnostic="ml_readiness", + ) + + from framevitals.ml_readiness import calculate_ml_readiness + from framevitals.profiler import build_profile + + dataframe = source.load() + dataset_profile = build_profile(dataframe) + return _named( + calculate_ml_readiness(dataframe, profile=dataset_profile), + metadata.name, + diagnostic="ml_readiness", + ) + + +def quality( + data: DataInput, + *, + max_sample_rows: int = 5_000, + max_columns: int = 100, + max_missingness_columns: int = 25, +) -> DiagnosticResult: + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.streaming_profile import build_streaming_profile + from framevitals.streaming_quality import run_streaming_quality_diagnostics + + dataset_profile, sample = build_streaming_profile( + source, + sample_rows=max_sample_rows, + return_sample=True, + ) + payload = run_streaming_quality_diagnostics( + sample, + profile=dataset_profile, + source_rows=int(metadata.rows or len(sample)), + source_columns=int(metadata.columns or len(sample.columns)), + max_sample_rows=max_sample_rows, + max_columns=max_columns, + max_missingness_columns=max_missingness_columns, + ) + return _named(payload, metadata.name, diagnostic="quality") + + from framevitals.column_roles import infer_column_roles + from framevitals.profiler import build_profile + from framevitals.quality_diagnostics import run_quality_diagnostics + + dataframe = source.load() + dataset_profile = build_profile(dataframe) + column_roles = infer_column_roles(dataframe) + payload = run_quality_diagnostics( + dataframe, + profile=dataset_profile, + column_roles=column_roles, + max_sample_rows=max_sample_rows, + max_columns=max_columns, + max_missingness_columns=max_missingness_columns, + ) + return _named(payload, metadata.name, diagnostic="quality") + + +def statistics( + data: DataInput, + *, + max_pairs: int = 20, + mode: str = "standard", +) -> DiagnosticResult: + """Run deep statistics through the same large-data budget as full analysis.""" + from framevitals.budgeted_analysis import run_budgeted_deep_statistics + from framevitals.execution import derive_execution_budget + + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.streaming_profile import sample_streaming_source + + if metadata.rows is None or metadata.columns is None: + raise ValueError("Streaming statistics require source shape metadata.") + + source_rows = int(metadata.rows) + source_columns = int(metadata.columns) + budget = derive_execution_budget(source_rows, source_columns, mode=mode) + sample_limit = max( + 20, + min( + budget.deep_statistics_sample_rows, + budget.bootstrap_sample_rows, + ), + ) + sample = sample_streaming_source(source, sample_rows=sample_limit) + payload = run_budgeted_deep_statistics( + sample, + budget=budget, + max_pairs=max_pairs, + ) + execution = dict(payload.get("execution", {})) + sampled = len(sample) < source_rows + execution.update({ + "sampled": sampled, + "source_rows": source_rows, + "source_columns": source_columns, + "sample_rows": int(len(sample)), + "strategy": ( + "streaming_stratified_jitter_global_rows" + if sampled + else "full_stream_via_batches" + ), + "full_materialization": False, + "reason": ( + "Deep statistics ran on a bounded deterministic stratified-jitter sample " + "selected directly from the streaming source." + if sampled + else "The streaming source fits within the deep-statistics execution budget." + ), + }) + payload["execution"] = normalize_execution( + execution, + method="bounded_deep_statistics", + full_materialization=False, + source=metadata.to_dict(), + ) + payload["source"] = metadata.to_dict() + return _named(payload, metadata.name, diagnostic="statistics") + + dataframe = source.load() + budget = derive_execution_budget( + len(dataframe), + len(dataframe.columns), + mode=mode, + ) + payload = run_budgeted_deep_statistics( + dataframe, + budget=budget, + max_pairs=max_pairs, + ) + payload["execution"] = normalize_execution( + payload.get("execution", {}), + method="bounded_deep_statistics", + full_materialization=load_fully_materializes(metadata), + source=metadata.to_dict(), + ) + payload["source"] = metadata.to_dict() + return _named(payload, metadata.name, diagnostic="statistics") + + +def anomalies( + data: DataInput, + *, + contamination: float = 0.05, + threshold: float = 0.6, + max_columns: int = 30, + top_k: int = 25, + mode: str = "standard", +) -> DiagnosticResult: + """Run anomaly diagnostics with bounded covariance/neighbor work.""" + from framevitals.budgeted_analysis import run_budgeted_anomalies + from framevitals.execution import derive_execution_budget + + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.streaming_profile import ( + numeric_columns_for_streaming_source, + sample_streaming_source, + ) + + if metadata.rows is None or metadata.columns is None: + raise ValueError("Streaming anomaly analysis requires source shape metadata.") + source_rows = int(metadata.rows) + source_columns = int(metadata.columns) + budget = derive_execution_budget(source_rows, source_columns, mode=mode) + numeric_columns = numeric_columns_for_streaming_source(source) + if not numeric_columns: + execution = execution_provenance( + "bounded_anomaly_detection", + full_materialization=False, + source=metadata.to_dict(), + sampled=False, + source_rows=source_rows, + source_columns=source_columns, + sample_rows=0, + strategy="streaming_schema_only", + scope="bounded_anomaly_detection", + extra={"projected_columns": 0}, + ) + return _named( + { + "available": False, + "reason": "No numeric columns available for anomaly analysis.", + "execution": execution, + "source": metadata.to_dict(), + }, + metadata.name, + diagnostic="anomalies", + ) + + sample_limit = max(100, int(budget.anomaly_sample_rows)) + sample = sample_streaming_source( + source, + sample_rows=sample_limit, + columns=numeric_columns, + ) + payload = run_budgeted_anomalies( + sample, + budget=budget, + contamination=contamination, + threshold=threshold, + max_columns=max_columns, + top_k=top_k, + ) + execution = dict(payload.get("execution", {})) + sampled = len(sample) < source_rows + execution.update({ + "sampled": sampled, + "source_rows": source_rows, + "source_columns": source_columns, + "projected_columns": int(len(numeric_columns)), + "sample_rows": int(len(sample)), + "strategy": ( + "streaming_stratified_jitter_numeric_projection" + if sampled + else "full_stream_numeric_projection" + ), + "full_materialization": False, + "reason": ( + "Anomaly diagnostics ran on a bounded deterministic stratified-jitter " + "numeric projection selected directly from the streaming source." + if sampled + else "The streaming source fits within the anomaly execution budget." + ), + }) + payload["execution"] = normalize_execution( + execution, + method="bounded_anomaly_detection", + full_materialization=False, + source=metadata.to_dict(), + ) + payload["source"] = metadata.to_dict() + return _named(payload, metadata.name, diagnostic="anomalies") + + dataframe = source.load() + budget = derive_execution_budget( + len(dataframe), + len(dataframe.columns), + mode=mode, + ) + payload = run_budgeted_anomalies( + dataframe, + budget=budget, + contamination=contamination, + threshold=threshold, + max_columns=max_columns, + top_k=top_k, + ) + payload["execution"] = normalize_execution( + payload.get("execution", {}), + method="bounded_anomaly_detection", + full_materialization=load_fully_materializes(metadata), + source=metadata.to_dict(), + ) + payload["source"] = metadata.to_dict() + return _named(payload, metadata.name, diagnostic="anomalies") + + +def relationships( + data: DataInput, + *, + max_sample_rows: int = 512, + projections: int = 64, + min_abs_correlation: float = 0.80, + max_candidate_pairs: int = 250_000, + max_edges_returned: int = 5_000, +) -> DiagnosticResult: + from framevitals.relationship_graph import build_numeric_relationship_graph + + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.streaming_profile import ( + numeric_columns_for_streaming_source, + sample_streaming_source, + ) + + numeric_columns = numeric_columns_for_streaming_source(source) + sample = sample_streaming_source( + source, + sample_rows=max_sample_rows, + columns=numeric_columns, + ) + payload = build_numeric_relationship_graph( + sample, + max_sample_rows=max_sample_rows, + projections=projections, + min_abs_correlation=min_abs_correlation, + max_candidate_pairs=max_candidate_pairs, + max_edges_returned=max_edges_returned, + ) + source_rows = int(metadata.rows or len(sample)) + source_columns = int(metadata.columns or len(sample.columns)) + sampled = source_rows > len(sample) + strategy = ( + "streaming_stratified_jitter_global_rows" + if sampled + else "full_stream_via_batches" + ) + sample_metadata = payload.setdefault("sample", {}) + sample_metadata.update({ + "source_rows": source_rows, + "sample_rows": int(len(sample)), + "sampled": sampled, + "full_materialization": False, + "strategy": strategy, + }) + payload["source"] = metadata.to_dict() + payload["streaming_source"] = True + payload["full_materialization"] = False + payload["execution"] = execution_provenance( + "bounded_relationship_graph", + full_materialization=False, + source=metadata.to_dict(), + sampled=sampled, + source_rows=source_rows, + source_columns=source_columns, + sample_rows=int(len(sample)), + strategy=strategy, + components={ + "numeric_projection": "schema_exact", + "relationship_candidates": ( + "bounded_row_sample" if sampled else "full_input" + ), + }, + extra={"projected_columns": int(len(numeric_columns))}, + ) + return _named(payload, metadata.name, diagnostic="relationships") + + dataframe = source.load() + payload = build_numeric_relationship_graph( + dataframe, + max_sample_rows=max_sample_rows, + projections=projections, + min_abs_correlation=min_abs_correlation, + max_candidate_pairs=max_candidate_pairs, + max_edges_returned=max_edges_returned, + ) + sample_metadata = payload.get("sample", {}) + if not isinstance(sample_metadata, dict): + sample_metadata = {} + sampled = bool(sample_metadata.get("sampled", False)) + sample_rows = int(sample_metadata.get("sample_rows", len(dataframe))) + strategy = str(sample_metadata.get("strategy", "full_input")) + payload["source"] = metadata.to_dict() + payload["full_materialization"] = load_fully_materializes(metadata) + payload["execution"] = execution_provenance( + "bounded_relationship_graph", + full_materialization=load_fully_materializes(metadata), + source=metadata.to_dict(), + sampled=sampled, + source_rows=int(len(dataframe)), + source_columns=int(len(dataframe.columns)), + sample_rows=sample_rows, + strategy=strategy, + components={ + "relationship_candidates": "bounded_row_sample" if sampled else "full_input", + }, + ) + return _named(payload, metadata.name, diagnostic="relationships") + + +def target_analysis( + data: DataInput, + *, + target: str, +) -> DiagnosticResult: + source = resolve_source(data) + metadata = source.inspect() + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.streaming_target import run_streaming_target_analysis + + payload = run_streaming_target_analysis(source, target=target) + return _named(payload, metadata.name, diagnostic="target_analysis") + + from framevitals.column_roles import infer_column_roles + from framevitals.target_intelligence import run_target_intelligence + + dataframe = source.load() + if target not in dataframe.columns: + raise ValueError(f"Target column not found: {target}") + column_roles = infer_column_roles(dataframe) + payload = run_target_intelligence( + dataframe, + target_column=target, + column_roles=column_roles, + ) + return _named(payload, metadata.name, diagnostic="target_analysis") diff --git a/src/framevitals/frontend_api.py b/src/framevitals/frontend_api.py index 6c2274c..e5a68ad 100644 --- a/src/framevitals/frontend_api.py +++ b/src/framevitals/frontend_api.py @@ -205,8 +205,8 @@ def build_dashboard_payload( health_score = result.get("health", {}).get("overall_score", 0) health_label = result.get("health", {}).get("label", "Unknown") - ml_score = result.get("ml_readiness", {}).get("score", 0) - ml_label = result.get("ml_readiness", {}).get("label", "Unknown") + result.get("ml_readiness", {}).get("score", 0) + result.get("ml_readiness", {}).get("label", "Unknown") payload = { "id": result.get("dataset_id"), diff --git a/src/framevitals/health_score.py b/src/framevitals/health_score.py index fc298be..71ec093 100644 --- a/src/framevitals/health_score.py +++ b/src/framevitals/health_score.py @@ -1,7 +1,14 @@ -import pandas as pd +from __future__ import annotations + +from typing import Any, Mapping + import numpy as np +import pandas as pd + +from framevitals.provenance import normalize_execution -def calculate_outlier_percent(df): + +def calculate_outlier_percent(df: pd.DataFrame): numeric_cols = df.select_dtypes(include=[np.number]).columns if len(numeric_cols) == 0: return 0.0, {} @@ -28,36 +35,40 @@ def calculate_outlier_percent(df): percent = round(outlier_count / max(total_cells, 1) * 100, 2) return percent, details -def calculate_health_score(df, profile): - rows = max(profile["shape"]["rows"], 1) - columns = max(profile["shape"]["columns"], 1) - missing_total = sum( - value for value in profile["missing_counts"].values() - if isinstance(value, int) - ) +def _health_label(overall: float) -> str: + if overall >= 90: + return "Excellent" + if overall >= 75: + return "Good" + if overall >= 60: + return "Moderate" + if overall >= 40: + return "Poor" + return "Critical" - missing_percent = missing_total / (rows * columns) * 100 - duplicate_percent = profile["duplicate_percent"] - outlier_percent, outlier_details = calculate_outlier_percent(df) - constant_columns = [] - high_cardinality_columns = [] - - for col in df.columns: - unique_count = df[col].nunique(dropna=True) - if unique_count <= 1: - constant_columns.append(col) - if df[col].dtype == "object" and unique_count > 0.8 * rows: - high_cardinality_columns.append(col) - - constant_percent = len(constant_columns) / columns * 100 - high_cardinality_percent = len(high_cardinality_columns) / columns * 100 +def _score_payload( + *, + missing_percent: float, + duplicate_percent: float, + outlier_percent: float, + constant_columns: list[str], + high_cardinality_columns: list[str], + columns: int, + outlier_details: Mapping[str, Any], + execution: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + constant_percent = len(constant_columns) / max(columns, 1) * 100 + high_cardinality_percent = len(high_cardinality_columns) / max(columns, 1) * 100 completeness_score = max(0, round(100 - missing_percent, 2)) uniqueness_score = max(0, round(100 - duplicate_percent, 2)) outlier_safety_score = max(0, round(100 - outlier_percent, 2)) - consistency_score = max(0, round(100 - constant_percent - high_cardinality_percent, 2)) + consistency_score = max( + 0, + round(100 - constant_percent - high_cardinality_percent, 2), + ) overall = round( completeness_score * 0.30 @@ -67,23 +78,11 @@ def calculate_health_score(df, profile): + 10, 2, ) - overall = max(0, min(100, overall)) - if overall >= 90: - label = "Excellent" - elif overall >= 75: - label = "Good" - elif overall >= 60: - label = "Moderate" - elif overall >= 40: - label = "Poor" - else: - label = "Critical" - - return { + payload: dict[str, Any] = { "overall_score": overall, - "label": label, + "label": _health_label(overall), "components": { "completeness": completeness_score, "consistency": consistency_score, @@ -96,6 +95,175 @@ def calculate_health_score(df, profile): "outlier_percent": outlier_percent, "constant_columns": constant_columns, "high_cardinality_columns": high_cardinality_columns, - "outlier_details": outlier_details, + "outlier_details": dict(outlier_details), }, } + if execution is not None: + payload["execution"] = dict(execution) + return payload + + +def _profile_constant_and_cardinality_columns( + profile: Mapping[str, Any], +) -> tuple[list[str], list[str]]: + """Infer consistency signals from a profile without raw full-row access.""" + rows = int(profile.get("shape", {}).get("rows", 0) or 0) + numeric_summary = profile.get("numeric_summary", {}) + categorical_summary = profile.get("categorical_summary", {}) + + constant_columns: list[str] = [] + high_cardinality_columns: list[str] = [] + + if isinstance(numeric_summary, Mapping): + for column, raw_summary in numeric_summary.items(): + if not isinstance(raw_summary, Mapping): + continue + count = int(raw_summary.get("count", 0) or 0) + minimum = raw_summary.get("min") + maximum = raw_summary.get("max") + if count > 0 and minimum is not None and maximum is not None and minimum == maximum: + constant_columns.append(str(column)) + + if isinstance(categorical_summary, Mapping): + for column, raw_summary in categorical_summary.items(): + if not isinstance(raw_summary, Mapping): + continue + unique_values = int(raw_summary.get("unique_values", 0) or 0) + if unique_values <= 1: + constant_columns.append(str(column)) + if rows > 0 and unique_values > 0.8 * rows: + high_cardinality_columns.append(str(column)) + + return sorted(set(constant_columns)), sorted(set(high_cardinality_columns)) + + +def calculate_health_score_from_profile_sample( + profile: Mapping[str, Any], + sample: pd.DataFrame, +) -> dict[str, Any]: + """Calculate a bounded health score from a full profile plus row sample. + + On ultra-wide sources the profile may cover all rows but a deterministic + subset of columns. Completeness and consistency are then estimates over that + projected column sample rather than being diluted by the unprofiled width. + """ + rows = max(int(profile.get("shape", {}).get("rows", 0) or 0), 1) + source_columns = max(int(profile.get("shape", {}).get("columns", 0) or 0), 1) + streaming_metadata = profile.get("streaming_metadata", {}) + profiled_columns = source_columns + if isinstance(streaming_metadata, Mapping): + profiled_columns = max( + int(streaming_metadata.get("profiled_columns", source_columns) or source_columns), + 1, + ) + column_sampled = profiled_columns < source_columns + + missing_counts = profile.get("missing_counts", {}) + missing_total = sum( + int(value) + for value in missing_counts.values() + if isinstance(value, (int, np.integer)) + ) if isinstance(missing_counts, Mapping) else 0 + missing_percent = missing_total / (rows * profiled_columns) * 100 + duplicate_percent = float(profile.get("duplicate_percent", 0.0) or 0.0) + + outlier_percent, sample_outlier_details = calculate_outlier_percent(sample) + sample_rows = len(sample) + outlier_scale = rows / max(sample_rows, 1) + outlier_details = { + column: int(round(count * outlier_scale)) + for column, count in sample_outlier_details.items() + } + constant_columns, high_cardinality_columns = _profile_constant_and_cardinality_columns( + profile + ) + + duplicate_metadata = profile.get("duplicate_metadata", {}) + duplicate_estimated = bool( + isinstance(duplicate_metadata, Mapping) and duplicate_metadata.get("sampled") + ) + + execution = normalize_execution( + { + "method": "streaming_profile_with_bounded_row_sample", + "source_rows": rows, + "source_columns": source_columns, + "profiled_columns": profiled_columns, + "column_sampled": column_sampled, + "sample_rows": sample_rows, + "sampled": sample_rows < rows or column_sampled, + "full_materialization": False, + "components": { + "completeness": ( + "full_rows_projected_columns_estimate" + if column_sampled + else "full_stream_exact" + ), + "uniqueness": ( + "projected_columns_row_sample_estimate" + if column_sampled + else ( + "full_stream_sample_estimate" + if duplicate_estimated + else "full_stream_exact" + ) + ), + "consistency": ( + "full_rows_projected_columns_estimate" + if column_sampled + else "full_stream_profile" + ), + "outlier_safety": ( + "projected_columns_bounded_row_sample_estimate" + if column_sampled + else ( + "bounded_row_sample_estimate" if sample_rows < rows else "exact" + ) + ), + }, + }, + method="streaming_profile_with_bounded_row_sample", + full_materialization=False, + ) + return _score_payload( + missing_percent=missing_percent, + duplicate_percent=duplicate_percent, + outlier_percent=outlier_percent, + constant_columns=constant_columns, + high_cardinality_columns=high_cardinality_columns, + columns=profiled_columns, + outlier_details=outlier_details, + execution=execution, + ) + + +def calculate_health_score(df: pd.DataFrame, profile: Mapping[str, Any]): + rows = max(int(profile["shape"]["rows"]), 1) + columns = max(int(profile["shape"]["columns"]), 1) + + missing_total = sum( + value for value in profile["missing_counts"].values() if isinstance(value, int) + ) + missing_percent = missing_total / (rows * columns) * 100 + duplicate_percent = profile["duplicate_percent"] + outlier_percent, outlier_details = calculate_outlier_percent(df) + + constant_columns = [] + high_cardinality_columns = [] + + for col in df.columns: + unique_count = df[col].nunique(dropna=True) + if unique_count <= 1: + constant_columns.append(col) + if df[col].dtype == "object" and unique_count > 0.8 * rows: + high_cardinality_columns.append(col) + + return _score_payload( + missing_percent=missing_percent, + duplicate_percent=duplicate_percent, + outlier_percent=outlier_percent, + constant_columns=constant_columns, + high_cardinality_columns=high_cardinality_columns, + columns=columns, + outlier_details=outlier_details, + ) diff --git a/src/framevitals/loader.py b/src/framevitals/loader.py index a36ab2b..9e55b7f 100644 --- a/src/framevitals/loader.py +++ b/src/framevitals/loader.py @@ -84,6 +84,17 @@ def _read_csv_tolerant( ) +def _read_excel(file_path: str | Path) -> pd.DataFrame: + """Read Excel input and surface a FrameVitals-specific dependency hint.""" + try: + return pd.read_excel(file_path) + except ImportError as exc: + raise ImportError( + "Excel input requires the optional FrameVitals Excel capability. " + 'Install it with: pip install "framevitals[excel]"' + ) from exc + + def load_dataset( file_path: str | Path, ) -> pd.DataFrame: @@ -91,7 +102,7 @@ def load_dataset( Load a supported tabular dataset. Supported formats: - CSV, TSV, XLSX, XLS, and JSON. + CSV, TSV, XLSX, XLS, and JSON. Excel readers are optional. """ path = Path(file_path) @@ -111,7 +122,7 @@ def load_dataset( ".xlsx", ".xls", }: - return pd.read_excel(path) + return _read_excel(path) if suffix == ".json": try: diff --git a/src/framevitals/ml_preprocessing.py b/src/framevitals/ml_preprocessing.py index 1b56d09..f1dd4d1 100644 --- a/src/framevitals/ml_preprocessing.py +++ b/src/framevitals/ml_preprocessing.py @@ -1,9 +1,14 @@ +"""Unified ML preprocessing used by every FrameVitals model workflow. + +The module deliberately keeps feature selection conservative: obvious IDs, +unsupported temporal columns, constants, and dangerously high-cardinality +categoricals are excluded before sklearn preprocessing. Numeric infinities are +converted to missing values so the standard median imputer can handle them. """ -Unified ML Preprocessing -========================= -Single source of truth for preparing features and target. -ALL ML modules MUST call prepare_ml_matrix() instead of their own prepare_xy(). -""" + +from __future__ import annotations + +import re import numpy as np import pandas as pd @@ -12,25 +17,79 @@ from sklearn.pipeline import Pipeline from sklearn.preprocessing import FunctionTransformer, OneHotEncoder, StandardScaler -_ID_KEYWORDS = [ - "id", "uuid", "hash", "key", "identifier", "index", - "roll", "roll_number", "rollno", "roll number", - "application", "registration", "serial", "ticket", - "account", "customer_id", "order_id", "transaction_id", - "userid", "studentid", "student_id", "user_id", -] - -_TIME_KEYWORDS = [ - "time", "date", "timestamp", "created", "updated", - "datetime", "ts", "event", -] _CATEGORICAL_DTYPES = ["object", "string", "category", "bool"] +_IDENTIFIER_TOKENS = {"id", "uuid", "hash", "identifier", "index"} +_IDENTIFIER_EXACT = { + "key", + "roll", + "roll_number", + "rollno", + "application", + "registration", + "serial", + "ticket", + "account", + "userid", + "studentid", +} +_TIME_TOKENS = {"time", "date", "timestamp", "datetime"} + + +def _normalise_column_name(name: object) -> tuple[str, set[str]]: + normalized = re.sub(r"[^a-z0-9]+", "_", str(name).strip().lower()).strip("_") + tokens = {token for token in normalized.split("_") if token} + return normalized, tokens + + +def _looks_like_identifier_name(name: object) -> bool: + """Detect identifier-like names without unsafe substring matching. + + The old ``"id" in column_name`` approach incorrectly classified ordinary + names such as ``paid_amount``. Token/boundary matching keeps common ID + conventions while avoiding those false positives. + """ + normalized, tokens = _normalise_column_name(name) + if normalized in _IDENTIFIER_EXACT: + return True + if tokens & _IDENTIFIER_TOKENS: + return True + if normalized.endswith("_key") or normalized.endswith("_number"): + prefix = normalized.rsplit("_", 1)[0] + if prefix in {"account", "serial", "ticket", "roll", "registration"}: + return True + return False + + +def _looks_like_time_name(name: object) -> bool: + normalized, tokens = _normalise_column_name(name) + if tokens & _TIME_TOKENS: + return True + if normalized in {"created", "updated", "created_at", "updated_at"}: + return True + if normalized.endswith("_at") and tokens & {"created", "updated"}: + return True + return False + + +def _is_categorical_dtype(series: pd.Series) -> bool: + return bool( + pd.api.types.is_object_dtype(series) + or pd.api.types.is_string_dtype(series.dtype) + or isinstance(series.dtype, pd.CategoricalDtype) + or pd.api.types.is_bool_dtype(series) + ) -def prepare_ml_matrix(df, target, drop_high_unique_ratio=0.95, min_non_missing=5): - """Prepare a clean feature matrix and target vector for ML.""" - warnings = [] +def prepare_ml_matrix( + df, + target, + drop_high_unique_ratio=0.95, + min_non_missing=5, + max_categorical_levels=200, +): + """Prepare a conservative feature matrix and target vector for ML.""" + warnings: list[str] = [] if target not in df.columns: return { @@ -40,49 +99,108 @@ def prepare_ml_matrix(df, target, drop_high_unique_ratio=0.95, min_non_missing=5 "categorical_features": [], "dropped_columns": [], "warnings": [f"Target '{target}' not found."], + "infinite_values_replaced": {}, "usable": False, } - data = df.dropna(subset=[target]).copy() + if not 0 < drop_high_unique_ratio <= 1: + raise ValueError("drop_high_unique_ratio must be in (0, 1].") + if min_non_missing < 1: + raise ValueError("min_non_missing must be at least 1.") + if max_categorical_levels < 2: + raise ValueError("max_categorical_levels must be at least 2.") + + data = df.copy() + + target_infinite_count = 0 + if pd.api.types.is_numeric_dtype(data[target]): + target_values = pd.to_numeric(data[target], errors="coerce") + target_array = target_values.to_numpy(dtype="float64", na_value=np.nan) + target_infinite_count = int(np.isinf(target_array).sum()) + if target_infinite_count: + data[target] = target_values.replace([np.inf, -np.inf], np.nan) + warnings.append( + f"Treated {target_infinite_count} infinite target values as missing." + ) + + data = data.dropna(subset=[target]).copy() rows_dropped = len(df) - len(data) if rows_dropped > 0: - warnings.append(f"Dropped {rows_dropped} rows with missing target values.") + warnings.append(f"Dropped {rows_dropped} rows with missing/invalid target values.") y = data[target] - X = data.drop(columns=[target]) + X = data.drop(columns=[target]).copy() + + dropped: list[dict[str, str]] = [] + dropped_names: set[str] = set() + infinite_values_replaced: dict[str, int] = {} + + def drop(column: str, reason: str) -> None: + if column not in dropped_names: + dropped.append({"column": column, "reason": reason}) + dropped_names.add(column) - dropped = [] - rows = max(len(X), 1) + for column in list(X.columns): + series = X[column] + if pd.api.types.is_numeric_dtype(series): + numeric = pd.to_numeric(series, errors="coerce") + values = numeric.to_numpy(dtype="float64", na_value=np.nan) + inf_count = int(np.isinf(values).sum()) + if inf_count: + X[column] = numeric.replace([np.inf, -np.inf], np.nan) + infinite_values_replaced[column] = inf_count + warnings.append( + f"Replaced {inf_count} infinite values in '{column}' with missing values for imputation." + ) - for col in list(X.columns): - non_missing = int(X[col].notna().sum()) - unique_count = X[col].nunique(dropna=True) - unique_ratio = unique_count / rows - lower = col.lower().replace("-", "_") + for column in list(X.columns): + series = X[column] + non_missing = int(series.notna().sum()) + unique_count = int(series.nunique(dropna=True)) + unique_ratio = unique_count / max(non_missing, 1) if non_missing < min_non_missing: - dropped.append({"column": col, "reason": "insufficient_non_missing"}) + drop(column, "insufficient_non_missing") continue if unique_count <= 1: - dropped.append({"column": col, "reason": "constant"}) + drop(column, "constant") continue - if any(kw in lower for kw in _ID_KEYWORDS): - dropped.append({"column": col, "reason": "id_like_keyword"}) + if _looks_like_identifier_name(column): + drop(column, "id_like_name") continue - if any(kw in lower for kw in _TIME_KEYWORDS) and not pd.api.types.is_numeric_dtype( - X[col] - ): - dropped.append({"column": col, "reason": "time_like_text"}) - continue - if unique_ratio > drop_high_unique_ratio and not pd.api.types.is_numeric_dtype(X[col]): - dropped.append({"column": col, "reason": "high_unique_non_numeric"}) + if _looks_like_time_name(column) and not pd.api.types.is_numeric_dtype(series): + drop(column, "time_like_non_numeric") continue - drop_cols = [d["column"] for d in dropped] - X = X.drop(columns=drop_cols, errors="ignore") + if _is_categorical_dtype(series): + if unique_count > max_categorical_levels: + drop(column, "high_cardinality_categorical") + continue + if unique_ratio > drop_high_unique_ratio: + drop(column, "high_unique_non_numeric") + continue + + if ( + not pd.api.types.is_numeric_dtype(series) + and not _is_categorical_dtype(series) + ): + drop(column, "unsupported_dtype") + + X = X.drop(columns=list(dropped_names), errors="ignore") numeric_features = X.select_dtypes(include=[np.number]).columns.tolist() - categorical_features = X.select_dtypes(include=_CATEGORICAL_DTYPES).columns.tolist() + categorical_features = [ + column + for column in X.columns + if _is_categorical_dtype(X[column]) + ] + + used_features = set(numeric_features) | set(categorical_features) + unused_columns = [column for column in X.columns if column not in used_features] + for column in unused_columns: + drop(column, "unsupported_dtype") + if unused_columns: + X = X.drop(columns=unused_columns, errors="ignore") total_features = len(numeric_features) + len(categorical_features) usable = total_features > 0 and len(y) >= 20 @@ -99,6 +217,9 @@ def prepare_ml_matrix(df, target, drop_high_unique_ratio=0.95, min_non_missing=5 "categorical_features": categorical_features, "dropped_columns": dropped, "warnings": warnings, + "infinite_values_replaced": infinite_values_replaced, + "target_infinite_values_dropped": target_infinite_count, + "max_categorical_levels": int(max_categorical_levels), "usable": usable, } diff --git a/src/framevitals/ml_readiness.py b/src/framevitals/ml_readiness.py index b32dce7..7feed77 100644 --- a/src/framevitals/ml_readiness.py +++ b/src/framevitals/ml_readiness.py @@ -1,33 +1,29 @@ -import numpy as np +"""ML-readiness scoring and compatibility helpers.""" +from __future__ import annotations -_CATEGORICAL_DTYPES = ["object", "string", "category", "bool"] +import sys +from types import ModuleType +from typing import Any +import numpy as np -def calculate_ml_readiness(df, profile=None): - """Calculate ML-readiness while reusing profile metrics when available. +from framevitals.provenance import normalize_execution - ``profile`` is optional to preserve the standalone helper API. The main - pipeline passes the already-built profile so FrameVitals does not rescan - the full dataset for missing values, duplicates, and basic column groups. - """ - rows, columns = df.shape - if profile is None: - missing_percent = float(df.isna().sum().sum() / max(rows * columns, 1) * 100) - duplicate_percent = float(df.duplicated().sum() / max(rows, 1) * 100) - categorical_cols = df.select_dtypes(include=_CATEGORICAL_DTYPES).columns.tolist() - numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist() - else: - missing_total = sum( - int(value) - for value in profile.get("missing_counts", {}).values() - if value is not None - ) - missing_percent = float(missing_total / max(rows * columns, 1) * 100) - duplicate_percent = float(profile.get("duplicate_percent", 0.0)) - categorical_cols = list(profile.get("categorical_columns", [])) - numeric_cols = list(profile.get("numeric_columns", [])) +_CATEGORICAL_DTYPES = ["object", "string", "category", "bool"] + + +def _score_ml_readiness( + *, + rows: int, + columns: int, + missing_total: int, + duplicate_percent: float, + categorical_cols: list[str], + numeric_cols: list[str], +) -> dict[str, Any]: + missing_percent = float(missing_total / max(rows * columns, 1) * 100) encoding_penalty = min(20, len(categorical_cols) * 2) missing_penalty = min(30, missing_percent) @@ -62,3 +58,150 @@ def calculate_ml_readiness(df, profile=None): "Select a clear target column for supervised learning.", ], } + + +def calculate_ml_readiness_from_profile(profile: dict[str, Any]) -> dict[str, Any]: + """Calculate ML-readiness directly from a completed dataset profile. + + Streaming ultra-wide profiles may intentionally cover all source rows but + only a deterministic projection of columns. In that case missingness and + column-group penalties must be normalized by the profiled width, not by the + unobserved source width. The result is explicitly marked as a projected- + column estimate rather than a full-schema exact score. + """ + shape = profile.get("shape", {}) + rows = int(shape.get("rows", 0) or 0) + source_columns = int(shape.get("columns", len(profile.get("columns", []))) or 0) + streaming = profile.get("streaming_metadata", {}) + profiled_columns = int( + streaming.get("profiled_columns", len(profile.get("columns", []))) + or len(profile.get("columns", [])) + ) + column_sampled = bool(streaming.get("column_sampled", False)) or ( + profiled_columns < source_columns + ) + scoring_columns = profiled_columns if column_sampled else source_columns + + missing_total = sum( + int(value) + for value in profile.get("missing_counts", {}).values() + if value is not None + ) + duplicate_percent = float(profile.get("duplicate_percent", 0.0)) + categorical_cols = list(profile.get("categorical_columns", [])) + numeric_cols = list(profile.get("numeric_columns", [])) + + result = _score_ml_readiness( + rows=rows, + columns=scoring_columns, + missing_total=missing_total, + duplicate_percent=duplicate_percent, + categorical_cols=categorical_cols, + numeric_cols=numeric_cols, + ) + + if column_sampled: + result["score_scope"] = "full_rows_projected_columns_estimate" + result["profiled_columns"] = profiled_columns + result["source_columns"] = source_columns + result["issues"]["missing_percent_scope"] = "profiled_columns" + result["issues"]["duplicate_percent_scope"] = ( + "bounded_rows_projected_columns" + ) + else: + result["score_scope"] = "full_schema" + + if streaming.get("enabled"): + duplicate_metadata = profile.get("duplicate_metadata", {}) + duplicate_sampled = bool(duplicate_metadata.get("sampled")) + if column_sampled: + missingness_scope = "full_rows_projected_columns_exact" + column_group_scope = "projected_schema_exact" + duplicate_scope = ( + "bounded_row_sample_projected_columns_estimate" + if duplicate_sampled + else "projected_columns_exact" + ) + else: + missingness_scope = "full_stream_exact" + column_group_scope = "schema_exact" + duplicate_scope = ( + "bounded_row_sample_estimate" if duplicate_sampled else "exact" + ) + + result["execution"] = normalize_execution( + { + "method": "streaming_profile", + "full_materialization": bool(streaming.get("full_materialization", False)), + "source_rows": rows, + "source_columns": source_columns, + "profiled_columns": profiled_columns, + "column_sampled": column_sampled, + "sample_rows": int(streaming.get("sample_rows", 0) or 0), + "sampled": int(streaming.get("sample_rows", 0) or 0) < rows, + "components": { + "missingness": missingness_scope, + "column_groups": column_group_scope, + "duplicate_rate": duplicate_scope, + }, + }, + method="streaming_profile", + full_materialization=bool(streaming.get("full_materialization", False)), + ) + + return result + + +def calculate_ml_readiness(df, profile=None): + """Calculate ML-readiness while reusing profile metrics when available. + + ``profile`` is optional to preserve the standalone helper API. The main + pipeline passes the already-built profile so FrameVitals does not rescan + the full dataset for missing values, duplicates, and basic column groups. + """ + rows, columns = df.shape + + if profile is None: + # Standalone compatibility path. The public focused API builds a profile + # first and therefore avoids repeating these scans. + missing_total = sum(int(df[column].isna().sum()) for column in df.columns) + duplicate_percent = float(df.duplicated().sum() / max(rows, 1) * 100) + categorical_cols = df.select_dtypes(include=_CATEGORICAL_DTYPES).columns.tolist() + numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist() + else: + missing_total = sum( + int(value) + for value in profile.get("missing_counts", {}).values() + if value is not None + ) + duplicate_percent = float(profile.get("duplicate_percent", 0.0)) + categorical_cols = list(profile.get("categorical_columns", [])) + numeric_cols = list(profile.get("numeric_columns", [])) + + return _score_ml_readiness( + rows=int(rows), + columns=int(columns), + missing_total=missing_total, + duplicate_percent=duplicate_percent, + categorical_cols=categorical_cols, + numeric_cols=numeric_cols, + ) + + +class _CallableReadinessModule(ModuleType): + """Preserve ``fv.ml_readiness(data)`` after importing this compatibility module. + + Python normally places an imported submodule on its parent package under the + same attribute name. Since FrameVitals historically also exposes the focused + function ``framevitals.ml_readiness(...)``, importing this module would + otherwise replace that function with a non-callable module object. Making + the compatibility module callable preserves both APIs without eager imports. + """ + + def __call__(self, data: Any) -> dict[str, Any]: + from framevitals.focused import ml_readiness as public_ml_readiness + + return public_ml_readiness(data) + + +sys.modules[__name__].__class__ = _CallableReadinessModule diff --git a/src/framevitals/model_leaderboard.py b/src/framevitals/model_leaderboard.py index 84cb9be..7bbfc53 100644 --- a/src/framevitals/model_leaderboard.py +++ b/src/framevitals/model_leaderboard.py @@ -1,30 +1,9 @@ -""" -Model Leaderboard (WS-3) -======================== -Cross-validated leaderboard of multiple ML models for a given target. - -For classification: - DummyClassifier, LogisticRegression, KNeighborsClassifier, - RandomForestClassifier, GradientBoostingClassifier, - XGBClassifier (optional), LGBMClassifier (optional) - -For regression: - DummyRegressor, Ridge, Lasso, KNeighborsRegressor, - RandomForestRegressor, GradientBoostingRegressor, - XGBRegressor (optional), LGBMRegressor (optional) - -Each model is wrapped in a Pipeline with the shared preprocessor from -modules.ml_preprocessing.prepare_ml_matrix + build_sklearn_preprocessor. +"""Cross-validated baseline model leaderboard for FrameVitals. -Validation: - StratifiedKFold(5) for classification, KFold(5) for regression. - XGBoost / LightGBM are imported lazily and skipped if missing. - -The winner is also fit on the full training data and a calibration check -(classification) or residual summary (regression) is computed on a hold-out. - -Public entry point: - run_model_leaderboard(df, target_column, task_type=None) -> dict +The leaderboard is a diagnostic for dataset learnability, not an AutoML system. +It compares several lightweight/classical models (plus optional XGBoost and +LightGBM), keeps preprocessing inside each CV fold, records failures, and +compares the best real model against a dummy baseline. """ from __future__ import annotations @@ -63,9 +42,6 @@ prepare_ml_matrix, ) -# --------------------------------------------------------------------------- -# Optional heavy models — imported lazily so missing libs don't break import -# --------------------------------------------------------------------------- def _maybe_xgb_classifier(): try: @@ -134,31 +110,25 @@ def _maybe_lgbm_regressor(): return None -# --------------------------------------------------------------------------- -# Task auto-detection -# --------------------------------------------------------------------------- - def _infer_task_type(y: pd.Series) -> str: """Best-effort: classification if dtype is non-numeric or low-cardinality.""" if pd.api.types.is_numeric_dtype(y): n = len(y) unique = int(y.nunique(dropna=True)) - # Heuristic: integer-like with few unique values is classification if unique <= 20 and unique <= max(2, int(n * 0.05)): return "classification" return "regression" return "classification" -# --------------------------------------------------------------------------- -# Model registries -# --------------------------------------------------------------------------- - def _classification_registry(class_count: int) -> dict[str, Any]: registry: dict[str, Any] = { "DummyClassifier": DummyClassifier(strategy="most_frequent"), "LogisticRegression": LogisticRegression( - max_iter=2000, n_jobs=-1, class_weight="balanced", random_state=42 + max_iter=2000, + n_jobs=-1, + class_weight="balanced", + random_state=42, ), "KNeighborsClassifier": KNeighborsClassifier(n_neighbors=7), "RandomForestClassifier": RandomForestClassifier( @@ -169,7 +139,9 @@ def _classification_registry(class_count: int) -> dict[str, Any]: class_weight="balanced", ), "GradientBoostingClassifier": GradientBoostingClassifier( - n_estimators=150, max_depth=4, random_state=42 + n_estimators=150, + max_depth=4, + random_state=42, ), } xgb = _maybe_xgb_classifier() @@ -188,10 +160,15 @@ def _regression_registry() -> dict[str, Any]: "Lasso": Lasso(alpha=0.01, max_iter=20000, random_state=42), "KNeighborsRegressor": KNeighborsRegressor(n_neighbors=7), "RandomForestRegressor": RandomForestRegressor( - n_estimators=200, max_depth=10, random_state=42, n_jobs=-1 + n_estimators=200, + max_depth=10, + random_state=42, + n_jobs=-1, ), "GradientBoostingRegressor": GradientBoostingRegressor( - n_estimators=150, max_depth=4, random_state=42 + n_estimators=150, + max_depth=4, + random_state=42, ), } xgb = _maybe_xgb_regressor() @@ -203,10 +180,6 @@ def _regression_registry() -> dict[str, Any]: return registry -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - def _safe_round(value, ndigits: int = 4) -> float | None: if value is None: return None @@ -220,55 +193,147 @@ def _safe_round(value, ndigits: int = 4) -> float | None: def _cv_for(task_type: str, y: pd.Series, n_splits: int = 5): + if n_splits < 2: + raise ValueError("n_splits must be at least 2.") + + n_rows = len(y) + if n_rows < 2: + raise ValueError("At least 2 target rows are required for cross-validation.") + if task_type == "classification": - # Reduce splits if smallest class is too tiny - min_class = int(y.value_counts().min()) if len(y) else 0 - actual_splits = max(2, min(n_splits, min_class)) - return StratifiedKFold(n_splits=actual_splits, shuffle=True, random_state=42), actual_splits - return KFold(n_splits=n_splits, shuffle=True, random_state=42), n_splits + counts = y.value_counts() + min_class = int(counts.min()) if len(counts) else 0 + if min_class < 2: + raise ValueError( + "Each classification class needs at least 2 rows for stratified cross-validation." + ) + actual_splits = min(n_splits, min_class) + return ( + StratifiedKFold( + n_splits=actual_splits, + shuffle=True, + random_state=42, + ), + actual_splits, + ) + + # R² is undefined for a test fold containing fewer than two observations. + # Because ML preprocessing already requires >=20 rows, capping KFold at + # floor(n/2) guarantees every test fold has at least two rows while still + # honouring smaller user-requested fold counts. + max_r2_splits = max(2, n_rows // 2) + actual_splits = min(n_splits, max_r2_splits) + if actual_splits < 2: + raise ValueError("Regression cross-validation needs at least 2 folds.") + return ( + KFold(n_splits=actual_splits, shuffle=True, random_state=42), + actual_splits, + ) def _scoring_for(task_type: str) -> tuple[dict[str, str], str]: if task_type == "classification": - scoring = { + return { "accuracy": "accuracy", "f1_weighted": "f1_weighted", "precision_weighted": "precision_weighted", "recall_weighted": "recall_weighted", - } - return scoring, "f1_weighted" - scoring = { + }, "f1_weighted" + return { "r2": "r2", "neg_mae": "neg_mean_absolute_error", "neg_rmse": "neg_root_mean_squared_error", - } - return scoring, "r2" + }, "r2" + + +def _json_safe_label(value: Any) -> Any: + """Preserve ordinary class-label types while keeping metadata JSON-safe.""" + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, pd.Timestamp): + return value.isoformat() + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return str(value) + + +def _encode_classification_target( + y: pd.Series, +) -> tuple[pd.Series, list[dict[str, Any]]]: + """Encode every classification target to stable consecutive integer labels.""" + # sort=False also supports heterogeneous object labels that cannot be + # ordered against one another (for example a mix of strings and integers). + codes, uniques = pd.factorize(y, sort=False) + encoded = pd.Series(codes, index=y.index, name=y.name, dtype="int64") + mapping = [ + { + "encoded": int(index), + "label": _json_safe_label(value), + } + for index, value in enumerate(uniques.tolist()) + ] + return encoded, mapping -def _coerce_classification_target(y: pd.Series) -> pd.Series: - """XGBoost requires integer-encoded class labels; encode safely.""" - if pd.api.types.is_numeric_dtype(y): - return y - return y.astype("category").cat.codes +def _set_estimator_jobs(estimator: Any, jobs: int) -> Any: + """Avoid nested process/thread explosions inside parallel CV folds.""" + try: + params = estimator.get_params(deep=False) + if "n_jobs" in params: + estimator.set_params(n_jobs=jobs) + except Exception: + pass + return estimator + + +def _adapt_knn_neighbors(registry: dict[str, Any], n_rows: int, n_splits: int) -> int: + largest_test_fold = int(np.ceil(n_rows / n_splits)) + smallest_train_fold = max(1, n_rows - largest_test_fold) + neighbors = max(1, min(7, smallest_train_fold)) + for name in ("KNeighborsClassifier", "KNeighborsRegressor"): + estimator = registry.get(name) + if estimator is not None: + try: + estimator.set_params(n_neighbors=neighbors) + except Exception: + pass + return neighbors + +def _score_stability(std: float | None) -> str: + if std is None: + return "unknown" + if std <= 0.02: + return "stable" + if std <= 0.05: + return "moderate" + return "variable" -# --------------------------------------------------------------------------- -# Hold-out diagnostics for the winner -# --------------------------------------------------------------------------- def _classification_holdout( - pipeline: Pipeline, X: pd.DataFrame, y: pd.Series -) -> dict: + pipeline: Pipeline, + X: pd.DataFrame, + y: pd.Series, +) -> dict[str, Any]: n = len(y) test_size = max(50, int(n * 0.25)) if n >= 200 else max(20, int(n * 0.25)) test_size = min(test_size, n - 20) if test_size <= 0: return {"available": False, "reason": "not enough rows for holdout"} - stratify = y if y.nunique() > 1 and y.value_counts().min() >= 2 else None + class_count = int(y.nunique()) + stratify = y if class_count > 1 and y.value_counts().min() >= 2 else None + if stratify is not None: + test_size = max(test_size, class_count) + if n - test_size < class_count: + stratify = None X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=test_size, random_state=42, stratify=stratify + X, + y, + test_size=test_size, + random_state=42, + stratify=stratify, ) pipeline.fit(X_train, y_train) @@ -279,7 +344,9 @@ def _classification_holdout( "n_train": int(len(X_train)), "n_test": int(len(X_test)), "accuracy": _safe_round(accuracy_score(y_test, y_pred)), - "f1_weighted": _safe_round(f1_score(y_test, y_pred, average="weighted", zero_division=0)), + "f1_weighted": _safe_round( + f1_score(y_test, y_pred, average="weighted", zero_division=0) + ), "precision_weighted": _safe_round( precision_score(y_test, y_pred, average="weighted", zero_division=0) ), @@ -288,23 +355,27 @@ def _classification_holdout( ), } - # Brier score for binary calibration try: if hasattr(pipeline, "predict_proba"): proba = pipeline.predict_proba(X_test) classes = pipeline.classes_ if hasattr(pipeline, "classes_") else None if proba.shape[1] == 2 and classes is not None: - # Binary: pick positive-class column pos_idx = 1 y_bin = (y_test == classes[pos_idx]).astype(int).values - out["brier_score"] = _safe_round(brier_score_loss(y_bin, proba[:, pos_idx])) + out["brier_score"] = _safe_round( + brier_score_loss(y_bin, proba[:, pos_idx]) + ) except Exception as exc: out["brier_error"] = str(exc) return out -def _regression_holdout(pipeline: Pipeline, X: pd.DataFrame, y: pd.Series) -> dict: +def _regression_holdout( + pipeline: Pipeline, + X: pd.DataFrame, + y: pd.Series, +) -> dict[str, Any]: n = len(y) test_size = max(50, int(n * 0.25)) if n >= 200 else max(20, int(n * 0.25)) test_size = min(test_size, n - 20) @@ -312,11 +383,13 @@ def _regression_holdout(pipeline: Pipeline, X: pd.DataFrame, y: pd.Series) -> di return {"available": False, "reason": "not enough rows for holdout"} X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=test_size, random_state=42 + X, + y, + test_size=test_size, + random_state=42, ) pipeline.fit(X_train, y_train) y_pred = pipeline.predict(X_test) - residuals = y_test.values - y_pred return { @@ -335,30 +408,25 @@ def _regression_holdout(pipeline: Pipeline, X: pd.DataFrame, y: pd.Series) -> di } -# --------------------------------------------------------------------------- -# Public entry point -# --------------------------------------------------------------------------- - def run_model_leaderboard( df: pd.DataFrame, target_column: str, task_type: str | None = None, n_splits: int = 5, -) -> dict: - """ - Run a CV-validated model leaderboard. - - Args: - df: Dataframe. - target_column: Target column name. - task_type: "classification" | "regression" | None (auto-detect). - n_splits: CV folds (capped by smallest class for classification). - - Returns: - JSON-safe dict with leaderboard rows, winner card, and holdout metrics. - """ + n_jobs: int = 1, +) -> dict[str, Any]: + """Run a CV-validated diagnostic model leaderboard.""" if not target_column or target_column not in df.columns: - return {"available": False, "message": f"Target '{target_column}' not in dataframe."} + return { + "available": False, + "message": f"Target '{target_column}' not in dataframe.", + } + if task_type is not None and task_type not in {"classification", "regression"}: + raise ValueError("task_type must be 'classification', 'regression', or None.") + if n_splits < 2: + raise ValueError("n_splits must be at least 2.") + if n_jobs == 0: + raise ValueError("n_jobs cannot be 0.") prep = prepare_ml_matrix(df, target=target_column) if not prep["usable"]: @@ -366,6 +434,7 @@ def run_model_leaderboard( "available": False, "message": "; ".join(prep["warnings"]) or "Insufficient data for ML.", "dropped_columns": prep["dropped_columns"], + "warnings": prep["warnings"], } X = prep["X"] @@ -376,28 +445,68 @@ def run_model_leaderboard( if task_type is None: task_type = _infer_task_type(y) - # XGBoost requires integer-encoded labels for classification - y_for_models = _coerce_classification_target(y) if task_type == "classification" else y + target_encoding: list[dict[str, Any]] | None = None + if task_type == "classification": + y_for_models, target_encoding = _encode_classification_target(y) + if y_for_models.nunique() < 2: + return {"available": False, "message": "Target has <2 classes."} + else: + if not pd.api.types.is_numeric_dtype(y): + return { + "available": False, + "message": "Regression requires a numeric target.", + } + y_for_models = pd.to_numeric(y, errors="coerce") + if not np.isfinite(y_for_models.to_numpy(dtype=float)).all(): + return { + "available": False, + "message": "Regression target contains non-finite values after preprocessing.", + } - if task_type == "classification" and y_for_models.nunique() < 2: - return {"available": False, "message": "Target has <2 classes."} + try: + cv, actual_splits = _cv_for( + task_type, + y_for_models, + n_splits=n_splits, + ) + except ValueError as exc: + return { + "available": False, + "task_type": task_type, + "target_column": target_column, + "message": str(exc), + } - cv, actual_splits = _cv_for(task_type, y_for_models, n_splits=n_splits) scoring, primary = _scoring_for(task_type) - if task_type == "classification": - registry = _classification_registry(class_count=int(y_for_models.nunique())) + registry = _classification_registry( + class_count=int(y_for_models.nunique()) + ) else: registry = _regression_registry() - leaderboard: list[dict] = [] + knn_neighbors = _adapt_knn_neighbors( + registry, + n_rows=len(y_for_models), + n_splits=actual_splits, + ) + for estimator in registry.values(): + _set_estimator_jobs(estimator, 1) + + leaderboard: list[dict[str, Any]] = [] with warnings.catch_warnings(): warnings.simplefilter("ignore") - for name, est in registry.items(): - preprocessor = build_sklearn_preprocessor(numeric_features, categorical_features) - pipeline = Pipeline([("pre", preprocessor), ("model", est)]) + for name, estimator in registry.items(): + preprocessor = build_sklearn_preprocessor( + numeric_features, + categorical_features, + ) + pipeline = Pipeline([ + ("pre", preprocessor), + ("model", estimator), + ]) t0 = time.perf_counter() try: @@ -407,95 +516,164 @@ def run_model_leaderboard( y_for_models, cv=cv, scoring=scoring, - n_jobs=-1, + n_jobs=n_jobs, return_train_score=False, error_score=np.nan, ) - fit_time_s = float(np.mean(cv_results["fit_time"])) row: dict[str, Any] = { "model": name, - "fit_time_s": _safe_round(fit_time_s, ndigits=3), - "cv_total_s": _safe_round(time.perf_counter() - t0, ndigits=3), + "fit_time_s": _safe_round( + np.nanmean(cv_results["fit_time"]), + ndigits=3, + ), + "cv_total_s": _safe_round( + time.perf_counter() - t0, + ndigits=3, + ), "n_splits": actual_splits, } - for metric_key in scoring.keys(): - arr = cv_results[f"test_{metric_key}"] + for metric_key in scoring: + arr = np.asarray(cv_results[f"test_{metric_key}"], dtype=float) + valid_folds = int(np.isfinite(arr).sum()) if metric_key.startswith("neg_"): clean_key = metric_key[4:] row[f"{clean_key}_mean"] = _safe_round(-np.nanmean(arr)) row[f"{clean_key}_std"] = _safe_round(np.nanstd(arr)) + row[f"{clean_key}_valid_folds"] = valid_folds else: row[f"{metric_key}_mean"] = _safe_round(np.nanmean(arr)) row[f"{metric_key}_std"] = _safe_round(np.nanstd(arr)) - row["primary_score"] = ( - row.get(f"{primary}_mean") - if not primary.startswith("neg_") - else row.get(f"{primary[4:]}_mean") - ) + row[f"{metric_key}_valid_folds"] = valid_folds + + row["primary_score"] = row.get(f"{primary}_mean") + row["primary_std"] = row.get(f"{primary}_std") + row["score_stability"] = _score_stability(row["primary_std"]) + if row["primary_score"] is None: + row["error"] = "No valid cross-validation score was produced." leaderboard.append(row) except Exception as exc: - leaderboard.append({"model": name, "error": str(exc)}) + leaderboard.append({ + "model": name, + "error": f"{type(exc).__name__}: {exc}", + }) - # Sort by primary metric (higher is better for accuracy/f1/r2; for MAE/RMSE we use neg_) - def _sort_key(row: dict) -> float: - val = row.get("primary_score") - return -float("inf") if val is None else float(val) + def _sort_key(row: dict[str, Any]) -> float: + value = row.get("primary_score") + return -float("inf") if value is None else float(value) leaderboard.sort(key=_sort_key, reverse=True) - # Pick best non-dummy model + successful_rows = [ + row for row in leaderboard if row.get("primary_score") is not None + ] + failed_rows = [row for row in leaderboard if row.get("primary_score") is None] + dummy_row = next( + (row for row in successful_rows if "Dummy" in row["model"]), + None, + ) candidates = [ - r for r in leaderboard - if r.get("primary_score") is not None and "Dummy" not in r["model"] + row for row in successful_rows if "Dummy" not in row["model"] ] + + base_payload: dict[str, Any] = { + "available": True, + "task_type": task_type, + "target_column": target_column, + "primary_metric": primary, + "n_rows": int(len(y_for_models)), + "n_features": len(numeric_features) + len(categorical_features), + "numeric_features": numeric_features, + "categorical_features": categorical_features, + "dropped_columns": prep["dropped_columns"], + "warnings": list(prep["warnings"]), + "target_encoding": target_encoding, + "cv": { + "requested_splits": int(n_splits), + "actual_splits": int(actual_splits), + "n_jobs": int(n_jobs), + "knn_neighbors": int(knn_neighbors), + }, + "models_succeeded": len(successful_rows), + "models_failed": len(failed_rows), + "model_failures": [ + {"model": row["model"], "error": row.get("error")} + for row in failed_rows + ], + "leaderboard": leaderboard, + "baseline": ( + { + "model": dummy_row["model"], + "primary_score": dummy_row["primary_score"], + } + if dummy_row is not None + else None + ), + } + if not candidates: return { - "available": True, - "task_type": task_type, - "target_column": target_column, - "n_rows": int(len(y)), - "n_features": len(numeric_features) + len(categorical_features), - "numeric_features": numeric_features, - "categorical_features": categorical_features, - "leaderboard": leaderboard, + **base_payload, "winner": None, "message": "No non-dummy model produced a usable score.", } winner = candidates[0] + baseline_score = ( + float(dummy_row["primary_score"]) + if dummy_row is not None and dummy_row.get("primary_score") is not None + else None + ) + winner_score = float(winner["primary_score"]) + lift = winner_score - baseline_score if baseline_score is not None else None + beats_baseline = bool(lift is not None and lift > 0) + + if baseline_score is not None and not beats_baseline: + base_payload["warnings"].append( + "Best non-dummy model did not outperform the dummy baseline on the primary CV metric." + ) - # Refit winner on full data and run holdout with warnings.catch_warnings(): warnings.simplefilter("ignore") - winner_estimator = registry[winner["model"]] - winner_preprocessor = build_sklearn_preprocessor(numeric_features, categorical_features) - winner_pipeline = Pipeline([("pre", winner_preprocessor), ("model", winner_estimator)]) + winner_preprocessor = build_sklearn_preprocessor( + numeric_features, + categorical_features, + ) + winner_pipeline = Pipeline([ + ("pre", winner_preprocessor), + ("model", winner_estimator), + ]) try: if task_type == "classification": - holdout = _classification_holdout(winner_pipeline, X, y_for_models) + holdout = _classification_holdout( + winner_pipeline, + X, + y_for_models, + ) else: - holdout = _regression_holdout(winner_pipeline, X, y_for_models) + holdout = _regression_holdout( + winner_pipeline, + X, + y_for_models, + ) except Exception as exc: - holdout = {"available": False, "error": str(exc)} + holdout = { + "available": False, + "error": f"{type(exc).__name__}: {exc}", + } return { - "available": True, - "task_type": task_type, - "target_column": target_column, - "primary_metric": primary, - "n_rows": int(len(y)), - "n_features": len(numeric_features) + len(categorical_features), - "numeric_features": numeric_features, - "categorical_features": categorical_features, - "dropped_columns": prep["dropped_columns"], - "warnings": prep["warnings"], - "leaderboard": leaderboard, + **base_payload, "winner": { "model": winner["model"], "primary_score": winner["primary_score"], - "fit_time_s": winner["fit_time_s"], + "primary_std": winner.get("primary_std"), + "score_stability": winner.get("score_stability"), + "fit_time_s": winner.get("fit_time_s"), + "baseline_score": _safe_round(baseline_score), + "lift_over_baseline": _safe_round(lift), + "beats_baseline": beats_baseline if baseline_score is not None else None, "holdout": holdout, }, } diff --git a/src/framevitals/neural_anomaly.py b/src/framevitals/neural_anomaly.py new file mode 100644 index 0000000..85bde9c --- /dev/null +++ b/src/framevitals/neural_anomaly.py @@ -0,0 +1,138 @@ +"""Small neural reconstruction detector for research-mode anomaly analysis. + +This intentionally uses the existing scikit-learn dependency instead of adding a +large deep-learning runtime. The network is bounded by rows, columns and epochs +and is designed as an additional nonlinear anomaly view, not as a replacement +for deterministic statistical checks. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd +from sklearn.impute import SimpleImputer +from sklearn.neural_network import MLPRegressor +from sklearn.preprocessing import StandardScaler + + +def _unit_scale(values: np.ndarray) -> np.ndarray: + values = np.asarray(values, dtype=float) + if values.size == 0: + return values + lo = float(np.min(values)) + hi = float(np.max(values)) + if not np.isfinite(lo) or not np.isfinite(hi) or hi - lo <= 1e-12: + return np.zeros_like(values) + return (values - lo) / (hi - lo) + + +def neural_reconstruction_anomalies( + dataframe: pd.DataFrame, + *, + max_rows: int = 3_000, + max_columns: int = 24, + max_iter: int = 35, + top_k: int = 25, + random_state: int = 42, +) -> dict[str, Any]: + """Score rows with a tiny autoencoder-like MLP reconstruction network.""" + if max_rows < 20: + raise ValueError("max_rows must be at least 20.") + if max_columns < 1: + raise ValueError("max_columns must be at least 1.") + if max_iter < 1: + raise ValueError("max_iter must be positive.") + if top_k < 1: + raise ValueError("top_k must be positive.") + + numeric = dataframe.select_dtypes(include=[np.number]).replace( + [np.inf, -np.inf], + np.nan, + ) + dropped_constant: list[str] = [] + for column in list(numeric.columns): + if numeric[column].nunique(dropna=True) <= 1: + numeric = numeric.drop(columns=[column]) + dropped_constant.append(str(column)) + + if numeric.shape[1] < 2 or len(numeric) < 20: + return { + "available": False, + "reason": "Need at least 20 rows and two non-constant numeric columns.", + "dropped_constant_columns": dropped_constant, + } + + variances = numeric.var(axis=0, skipna=True).fillna(0.0) + selected = variances.sort_values(ascending=False).head(max_columns).index.tolist() + numeric = numeric[selected] + + source_rows = len(numeric) + if source_rows > max_rows: + positions = np.linspace(0, source_rows - 1, num=max_rows, dtype=np.int64) + positions = np.unique(positions) + work = numeric.iloc[positions] + sampled = True + else: + work = numeric + sampled = False + + imputer = SimpleImputer(strategy="median") + scaler = StandardScaler() + matrix = imputer.fit_transform(work) + matrix = scaler.fit_transform(matrix) + + n_features = matrix.shape[1] + bottleneck = max(2, min(12, n_features // 3 if n_features >= 6 else n_features - 1)) + hidden = max(bottleneck + 1, min(24, max(4, n_features // 2))) + network = MLPRegressor( + hidden_layer_sizes=(hidden, bottleneck, hidden), + activation="relu", + solver="adam", + alpha=1e-4, + batch_size=min(256, max(16, len(matrix) // 10)), + learning_rate_init=1e-3, + max_iter=max_iter, + early_stopping=True, + validation_fraction=0.1, + n_iter_no_change=5, + random_state=random_state, + ) + network.fit(matrix, matrix) + reconstructed = network.predict(matrix) + errors = np.mean((matrix - reconstructed) ** 2, axis=1) + scores = _unit_scale(errors) + + order = np.argsort(scores)[::-1][: min(top_k, len(scores))] + top_rows = [ + { + "row_index": str(work.index[int(index)]), + "score": round(float(scores[int(index)]), 6), + "reconstruction_error": round(float(errors[int(index)]), 6), + } + for index in order + ] + + return { + "available": True, + "method": "bounded_mlp_reconstruction", + "source_rows": int(source_rows), + "sample_rows": int(len(work)), + "sampled": sampled, + "used_columns": [str(column) for column in selected], + "columns_available": int(dataframe.select_dtypes(include=[np.number]).shape[1]), + "columns_used": int(len(selected)), + "truncated_columns": bool(len(selected) < dataframe.select_dtypes(include=[np.number]).shape[1]), + "architecture": [int(n_features), int(hidden), int(bottleneck), int(hidden), int(n_features)], + "iterations": int(network.n_iter_), + "loss": round(float(network.loss_), 6), + "score_summary": { + "mean": round(float(np.mean(scores)), 6), + "p95": round(float(np.quantile(scores, 0.95)), 6), + "p99": round(float(np.quantile(scores, 0.99)), 6), + "max": round(float(np.max(scores)), 6), + }, + "top_rows": top_rows, + "dropped_constant_columns": dropped_constant, + } diff --git a/src/framevitals/operations.py b/src/framevitals/operations.py new file mode 100644 index 0000000..deb665a --- /dev/null +++ b/src/framevitals/operations.py @@ -0,0 +1,440 @@ +"""Lightweight public data operations that do not require the full pipeline. + +Cleaning, contracts, drift, and quality gates are useful independently of EDA, +modeling, explainability, and report generation. Keeping them here prevents a +simple validation or comparison call from importing the heavy analysis stack. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +import pandas as pd + +from framevitals.cleaning_plan import ( + CleaningPlan, + apply_cleaning_plan, + infer_cleaning_plan, +) +from framevitals.contracts import infer_contract as _infer_contract +from framevitals.contracts import validate_contract +from framevitals.drift_analysis import compare_datasets, severity_at_least +from framevitals.profiler import build_profile +from framevitals.provenance import execution_provenance, load_fully_materializes +from framevitals.quality_results import DriftResult, GateResult, ValidationResult +from framevitals.sources import DatasetMetadata, StreamingDatasetSource, resolve_source + + +DataInput = Any +_DRIFT_SEVERITY_RANK = {"stable": 0, "minor": 1, "moderate": 2, "severe": 3} +_DRIFT_SAMPLE_ROWS = 50_000 + + +def _resolve_input(value: DataInput, *, label: str): + try: + source = resolve_source(value) + metadata = source.inspect() + except (TypeError, ValueError, FileNotFoundError) as exc: + if label == "Dataset": + raise + message = str(exc).replace("Dataset", label, 1) + raise type(exc)(message) from exc + + if metadata.rows == 0: + if metadata.kind == "memory": + raise ValueError(f"{label} DataFrame is empty.") + raise ValueError(f"{label} dataset is empty: {metadata.name}") + return source, metadata + + +def _load_input(value: DataInput, *, label: str) -> tuple[pd.DataFrame, str]: + source, metadata = _resolve_input(value, label=label) + dataframe = source.load() + if dataframe.empty: + if metadata.kind == "memory": + raise ValueError(f"{label} DataFrame is empty.") + raise ValueError(f"{label} dataset is empty: {metadata.name}") + return dataframe, metadata.name + + +def _comparison_frame( + source, + metadata: DatasetMetadata, + *, + sample_rows: int = _DRIFT_SAMPLE_ROWS, +) -> tuple[pd.DataFrame, dict[str, Any]]: + """Return drift input plus transparent source/materialization metadata.""" + if metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + from framevitals.streaming_profile import sample_streaming_source + + if metadata.rows is None: + raise ValueError("Streaming drift comparison requires a source row count.") + sample = sample_streaming_source(source, sample_rows=sample_rows) + source_rows = int(metadata.rows) + sampled = len(sample) < source_rows + strategy = ( + "streaming_evenly_spaced_global_rows" + if sampled + else "full_stream_via_batches" + ) + execution = execution_provenance( + "streaming_source_compare_input", + full_materialization=False, + source=metadata.to_dict(), + sampled=sampled, + source_rows=source_rows, + source_columns=int(metadata.columns or len(sample.columns)), + sample_rows=int(len(sample)), + strategy=strategy, + ) + return sample, execution + + dataframe = source.load() + if dataframe.empty: + raise ValueError(f"Dataset is empty: {metadata.name}") + execution = execution_provenance( + "full_compare_input", + full_materialization=load_fully_materializes(metadata), + source=metadata.to_dict(), + sampled=False, + source_rows=int(len(dataframe)), + source_columns=int(len(dataframe.columns)), + sample_rows=int(len(dataframe)), + strategy="full_input", + ) + return dataframe, execution + + +def _true_shape(execution: Mapping[str, Any]) -> list[int]: + return [ + int(execution.get("source_rows", 0)), + int(execution.get("source_columns", 0)), + ] + + +def _row_count_change_percent(reference_rows: int, current_rows: int) -> float | None: + if reference_rows <= 0: + return None + return round((current_rows - reference_rows) / reference_rows * 100, 4) + + +def plan_cleaning(data: DataInput) -> CleaningPlan: + """Infer a conservative cleaning plan without modifying the input data.""" + dataframe, _ = _load_input(data, label="Dataset") + dataset_profile = build_profile(dataframe) + return infer_cleaning_plan(dataframe, profile=dataset_profile) + + +def clean( + data: DataInput, + *, + plan: Mapping[str, Any] | None = None, +) -> pd.DataFrame: + """Return an explicitly cleaned copy; never mutate the caller's input.""" + dataframe, _ = _load_input(data, label="Dataset") + resolved_plan = plan if plan is not None else infer_cleaning_plan(dataframe) + return apply_cleaning_plan(dataframe, resolved_plan, copy=True) + + +def compare( + reference: DataInput, + current: DataInput, + *, + columns: list[str] | None = None, + max_columns: int = 30, +) -> DriftResult: + """Compare datasets, bounding value-distribution work for streaming sources.""" + if max_columns < 1: + raise ValueError("max_columns must be at least 1.") + + reference_source, reference_metadata = _resolve_input(reference, label="Reference") + current_source, current_metadata = _resolve_input(current, label="Current") + reference_df, reference_execution = _comparison_frame( + reference_source, + reference_metadata, + ) + current_df, current_execution = _comparison_frame( + current_source, + current_metadata, + ) + + payload = compare_datasets( + reference_df, + current_df, + columns=columns, + max_columns=max_columns, + ) + payload["reference_name"] = reference_metadata.name + payload["current_name"] = current_metadata.name + payload["ref_shape"] = _true_shape(reference_execution) + payload["cur_shape"] = _true_shape(current_execution) + payload["row_count_change_percent"] = _row_count_change_percent( + reference_execution["source_rows"], + current_execution["source_rows"], + ) + + any_sampled = bool( + reference_execution["sampled"] or current_execution["sampled"] + ) + components = { + "source_shape": "exact", + "schema_columns": "exact", + "value_distributions": "bounded_row_sample" if any_sampled else "full_input", + "missingness": "bounded_row_sample" if any_sampled else "full_input", + } + payload["execution"] = execution_provenance( + "bounded_source_compare" if any_sampled else "full_compare", + full_materialization=bool( + reference_execution["full_materialization"] + or current_execution["full_materialization"] + ), + sampled=any_sampled, + components=components, + extra={ + "sample_limit_rows_per_source": _DRIFT_SAMPLE_ROWS, + "reference": reference_execution, + "current": current_execution, + }, + ) + return DriftResult(payload) + + +def infer_contract( + data: DataInput, + *, + numeric_tolerance: float = 0.05, + max_categories: int = 20, + null_fraction_tolerance: float = 0.05, + infer_unique: bool = True, + min_unique_rows: int = 20, + allow_extra_columns: bool = False, +) -> dict[str, Any]: + """Infer a JSON-serializable contract from a reference dataset.""" + dataframe, source_name = _load_input(data, label="Reference") + contract = _infer_contract( + dataframe, + numeric_tolerance=numeric_tolerance, + max_categories=max_categories, + null_fraction_tolerance=null_fraction_tolerance, + infer_unique=infer_unique, + min_unique_rows=min_unique_rows, + allow_extra_columns=allow_extra_columns, + ) + contract["reference_name"] = source_name + return contract + + +def validate( + data: DataInput, + contract: Mapping[str, Any], +) -> ValidationResult: + """Validate a dataset against an inferred or explicit contract.""" + source, metadata = _resolve_input(data, label="Dataset") + dataframe = source.load() + if dataframe.empty: + raise ValueError(f"Dataset is empty: {metadata.name}") + payload = validate_contract(dataframe, contract) + payload["dataset_name"] = metadata.name + payload["execution"] = execution_provenance( + "exact_contract_validation", + full_materialization=load_fully_materializes(metadata), + source=metadata.to_dict(), + sampled=False, + source_rows=metadata.rows, + source_columns=metadata.columns, + reason=( + "Contract validation remains exact; uniqueness, allowed-value, and bound " + "constraints are not silently downgraded to sampled checks." + ), + ) + return ValidationResult(payload) + + +def gate( + current: DataInput, + *, + reference: DataInput | None = None, + contract: Mapping[str, Any] | None = None, + custom_checks: Sequence[Any] | None = None, + columns: list[str] | None = None, + max_columns: int = 30, + drift_warn_on: str = "moderate", + drift_fail_on: str = "severe", + fail_on_validation_warning: bool = False, +) -> GateResult: + """Combine contracts, custom invariants, and bounded drift into one verdict.""" + if reference is None and contract is None and not custom_checks: + raise ValueError( + "gate requires at least one of reference=, contract=, or custom_checks=." + ) + if drift_warn_on not in _DRIFT_SEVERITY_RANK: + raise ValueError("drift_warn_on must be one of: stable, minor, moderate, severe.") + if drift_fail_on not in _DRIFT_SEVERITY_RANK: + raise ValueError("drift_fail_on must be one of: stable, minor, moderate, severe.") + if _DRIFT_SEVERITY_RANK[drift_warn_on] > _DRIFT_SEVERITY_RANK[drift_fail_on]: + raise ValueError("drift_warn_on cannot be more severe than drift_fail_on.") + if max_columns < 1: + raise ValueError("max_columns must be at least 1.") + + _, current_metadata = _resolve_input(current, label="Current") + current_name = current_metadata.name + checks: dict[str, Any] = {} + reasons: list[str] = [] + status = "pass" + + def warn() -> None: + nonlocal status + if status == "pass": + status = "warn" + + def fail() -> None: + nonlocal status + status = "fail" + + if contract is not None: + validation_payload = validate(current, contract) + checks["validation"] = validation_payload + + validation_status = validation_payload.get("status") + if validation_status == "fail": + fail() + error_count = validation_payload.get("summary", {}).get("errors", 0) + reasons.append(f"Contract validation failed with {error_count} error(s).") + elif validation_status == "warn": + warning_count = validation_payload.get("summary", {}).get("warnings", 0) + if fail_on_validation_warning: + fail() + reasons.append( + f"Contract validation produced {warning_count} warning(s), promoted to failure." + ) + else: + warn() + reasons.append(f"Contract validation produced {warning_count} warning(s).") + + if reference is not None: + drift_payload = compare( + reference, + current, + columns=columns, + max_columns=max_columns, + ) + checks["drift"] = drift_payload + + if not drift_payload.get("available"): + fail() + reasons.append( + "Drift comparison was requested but could not produce a comparable result: " + f"{drift_payload.get('reason', 'unknown reason')}." + ) + else: + drift_severity = str( + drift_payload.get("gate", {}).get("severity", "unknown") + ) + if drift_severity not in _DRIFT_SEVERITY_RANK: + fail() + reasons.append( + "Drift comparison did not produce a recognized severity verdict." + ) + elif severity_at_least(drift_severity, drift_fail_on): + fail() + reasons.append( + f"Drift severity {drift_severity} reached fail threshold {drift_fail_on}." + ) + elif severity_at_least(drift_severity, drift_warn_on): + warn() + reasons.append( + f"Drift severity {drift_severity} reached warning threshold {drift_warn_on}." + ) + + drift_reasons = drift_payload.get("gate", {}).get("reasons", []) + if isinstance(drift_reasons, list): + for reason in drift_reasons[:10]: + text = str(reason) + if text and text not in reasons: + reasons.append(text) + + if custom_checks: + from framevitals.checks import run_checks + + custom_payload = run_checks(current, custom_checks) + checks["custom"] = custom_payload + custom_status = custom_payload.get("status") + if custom_status == "fail": + fail() + elif custom_status == "warn": + warn() + + for result in custom_payload.get("results", [])[:10]: + if not isinstance(result, Mapping) or result.get("passed"): + continue + message = str(result.get("message") or result.get("name") or "Custom check failed.") + if message and message not in reasons: + reasons.append(message) + + validation_execution = ( + checks.get("validation", {}).get("execution") + if isinstance(checks.get("validation"), Mapping) + else None + ) + drift_execution = ( + checks.get("drift", {}).get("execution") + if isinstance(checks.get("drift"), Mapping) + else None + ) + custom_execution = ( + checks.get("custom", {}).get("execution") + if isinstance(checks.get("custom"), Mapping) + else None + ) + execution_blocks = [ + block + for block in (validation_execution, drift_execution, custom_execution) + if isinstance(block, Mapping) + ] + full_materialization = any( + bool(block.get("full_materialization")) for block in execution_blocks + ) + + gate_execution = execution_provenance( + "quality_gate", + full_materialization=full_materialization, + source=current_metadata.to_dict(), + components={ + "validation": ( + validation_execution.get("method") + if isinstance(validation_execution, Mapping) + else None + ), + "drift": ( + drift_execution.get("method") + if isinstance(drift_execution, Mapping) + else None + ), + "custom": ( + custom_execution.get("method") + if isinstance(custom_execution, Mapping) + else None + ), + }, + extra={ + "validation": validation_execution, + "drift": drift_execution, + "custom": custom_execution, + }, + ) + + return GateResult({ + "status": status, + "passed": status != "fail", + "current_name": current_name, + "checks_run": list(checks), + "thresholds": { + "drift_warn_on": drift_warn_on, + "drift_fail_on": drift_fail_on, + "fail_on_validation_warning": bool(fail_on_validation_warning), + }, + "reasons": reasons, + "checks": checks, + "execution": gate_execution, + }) diff --git a/src/framevitals/pdf_report_builder.py b/src/framevitals/pdf_report_builder.py index 76396ce..45b6f8f 100644 --- a/src/framevitals/pdf_report_builder.py +++ b/src/framevitals/pdf_report_builder.py @@ -1,7 +1,7 @@ """ PDF Report Builder (v3 — editorial dark) ======================================== -Generates the DataLens AI dataset report PDF. +Generates the FrameVitals dataset report PDF. Visual identity matches the dashboard: cream + teal on near-black, heavy display weights, mono eyebrows, generous whitespace. Pages flow as: @@ -238,7 +238,7 @@ def _draw_header(self): # Eyebrow brand self.lay.text( MARGIN_L, PAGE_H - 0.42, - "DATALENS · AI DATASET REPORT", + "FRAMEVITALS · DATASET REPORT", fontsize=8, color=INK_3, fontweight="bold", family="monospace", va="center", ha="left", ) @@ -267,7 +267,7 @@ def _draw_footer(self): )) self.lay.text( MARGIN_L, MARGIN_B - 0.32, - "DATALENS · AI", + "FRAMEVITALS", fontsize=8, color=INK_3, fontweight="bold", family="monospace", va="center", ha="left", ) @@ -624,7 +624,6 @@ def add_chart_image(self, chart: dict, *, x: float, y_top: float, def _cover_page(c: Composer, result: dict): c.new_page(cover=True, chrome=False) - f = c.fig lay = c.lay # Hero band — gradient from teal-soft → near-black using a stack of rects @@ -639,7 +638,7 @@ def _cover_page(c: Composer, result: dict): # Brand eyebrow lay.text( MARGIN_L, PAGE_H - 1.0, - "DATALENS · AI", + "FRAMEVITALS", fontsize=10, color=ACCENT, fontweight="bold", family="monospace", ha="left", va="top", ) @@ -733,7 +732,7 @@ def _color_for_score(v): c.y = MARGIN_B + 1.6 lay.text( MARGIN_L, c.y, - "Generated by DataLens AI's analytical pipeline.", + "Generated by FrameVitals' analytical pipeline.", fontsize=10, color=INK_3, fontweight="700", ha="left", va="top", ) @@ -748,7 +747,7 @@ def _color_for_score(v): c.hline(MARGIN_L, MARGIN_L + CONTENT_W, MARGIN_B + 0.4, color=LINE, lw=0.6) lay.text( MARGIN_L, MARGIN_B + 0.18, - "DATALENS · AI", + "FRAMEVITALS", fontsize=8, color=INK_3, fontweight="bold", family="monospace", ha="left", va="top", ) @@ -830,7 +829,7 @@ def _render_executive_summary(c: Composer, result: dict): lead = ( f"This dataset contains {rows_v} rows across {cols_v} columns. " f"It scores {health_v}/100 on quality ({health_label}) and {ml_v}/100 on ML readiness ({ml_label}). " - f"DataLens AI ran in {str(mode).upper()} mode and selected " + f"FrameVitals ran in {str(mode).upper()} mode and selected " f"{sel.get('selected_count', 0)} analyses, recommended {sel.get('recommended_count', 0)} more, " f"and skipped {sel.get('skipped_count', 0)}." ) @@ -929,7 +928,7 @@ def _render_cleaning(c: Composer, result: dict): return c.new_page() c.section("Cleaning Summary", kind="cleaning", - subtitle="What changed when DataLens auto-cleaned the dataset.") + subtitle="Summary of applied cleaning actions and health impact.") before = (cl.get("before_health") or {}).get("overall_score") after = (cl.get("after_health") or {}).get("overall_score") @@ -1094,7 +1093,7 @@ def _render_ai_report(c: Composer, result: dict): c.new_page() c.section("AI Analyst Report", kind="ai", - subtitle="Narrative interpretation of the dataset by DataLens's analyst model.") + subtitle="Narrative interpretation of the dataset by FrameVitals' analyst model.") c.callout(f"source: {ai.get('source', 'unknown')}", color=ACCENT_2) c.text_block(text[:8000], size=10, color=INK_2, weight="700", line_height=0.20) @@ -1162,7 +1161,7 @@ def _build_toc_sections(result: dict) -> list[tuple[str, str, str]]: "Auto-detected issues, sorted by severity.")) if result.get("cleaning"): sections.append((f"{len(sections)+1:02d}", "Cleaning Summary", - "Before / after health and the actions DataLens took.")) + "Before / after health and applied cleaning actions.")) if result.get("deep_statistics_v2"): sections.append((f"{len(sections)+1:02d}", "Deep Statistics", "Pairwise relationships and group differences.")) @@ -1178,7 +1177,7 @@ def _build_toc_sections(result: dict) -> list[tuple[str, str, str]]: ai = result.get("ai_report") or {} if ai.get("text"): sections.append((f"{len(sections)+1:02d}", "AI Analyst Report", - "Narrative interpretation by the DataLens analyst model.")) + "Narrative interpretation by the FrameVitals analyst model.")) if result.get("charts"): sections.append((f"{len(sections)+1:02d}", "Visual Evidence", f"{len(result.get('charts') or [])} planner-driven charts, two per page.")) @@ -1188,7 +1187,7 @@ def _build_toc_sections(result: dict) -> list[tuple[str, str, str]]: def generate_pdf_report( result: dict, output_dir: Path | str = "reports", - report_title: str = "DataLens AI Dataset Report", + report_title: str = "FrameVitals Dataset Report", ) -> Path: output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/framevitals/pipeline.py b/src/framevitals/pipeline.py index 85f8b35..0486af0 100644 --- a/src/framevitals/pipeline.py +++ b/src/framevitals/pipeline.py @@ -1,8 +1,10 @@ """FrameVitals analysis pipeline. -Runs the full analytics stack in phases and parallelizes independent analyses -where safe. Optional failures are converted into structured error payloads so a -single diagnostic does not sink the whole report. +Runs the analytics stack in phases and parallelizes independent analyses where +safe. Optional failures are converted into structured error payloads so a +single diagnostic does not sink the whole report. Expensive modules can be +explicitly disabled through ``AnalysisConfig`` while adaptive execution budgets +bound dangerous work on large or wide datasets. """ from __future__ import annotations @@ -18,21 +20,25 @@ from framevitals.advanced_indicators import calculate_advanced_indicators from framevitals.ai_insights import generate_ai_report from framevitals.analysis_selector import select_analyses -from framevitals.anomaly_ensemble import detect_anomalies_ensemble +from framevitals.budgeted_analysis import ( + run_budgeted_anomalies, + run_budgeted_deep_statistics, + run_budgeted_time_series, +) from framevitals.cleaner import create_cleaned_dataset from framevitals.column_roles import infer_column_roles, summarize_roles +from framevitals.config import VALID_MODULES from framevitals.dataset_signals import detect_dataset_signals -from framevitals.deep_statistics_v2 import run_deep_statistics_v2 -from framevitals.explainability import explain_winner +from framevitals.execution import derive_execution_budget from framevitals.health_score import calculate_health_score from framevitals.loader import load_dataset from framevitals.ml_readiness import calculate_ml_readiness from framevitals.model_leaderboard import run_model_leaderboard from framevitals.profiler import build_profile +from framevitals.quality_diagnostics import run_quality_diagnostics from framevitals.signal_engine import build_signals +from framevitals.target_intelligence import run_target_intelligence from framevitals.text_profile import profile_text_columns -from framevitals.time_series import detect_and_analyze_time_series -from framevitals.visualizer import generate_charts logger = logging.getLogger("framevitals.pipeline") @@ -52,6 +58,21 @@ def _safe_call(name: str, fn: Callable[[], Any]) -> tuple[str, Any, float]: return name, value, elapsed_ms +def _skipped_module(module: str, reason: str = "Disabled by configuration.") -> dict: + return { + "available": False, + "skipped": True, + "module": module, + "reason": reason, + } + + +def _result_status(value: Any) -> str: + if isinstance(value, dict) and value.get("error"): + return "error" + return "ran" + + def run_full_analysis( dataset_id: str, file_path=None, @@ -62,35 +83,25 @@ def run_full_analysis( parallel_workers: int = 4, dataframe: pd.DataFrame | None = None, write_artifacts: bool = True, + disabled_modules: tuple[str, ...] | list[str] | set[str] | None = None, ) -> dict: - """Run the complete FrameVitals analysis pipeline. - - Parameters - ---------- - dataset_id: - Unique identifier used for optional artifact filenames. - file_path: - Path-like dataset source. Omit when ``dataframe`` is supplied. - original_filename: - Source label included in the result. - analysis_mode: - One of ``quick``, ``standard``, ``deep``, or ``research``. - skip_ai: - Skip the optional LLM report. - target_column: - Optional supervised-learning target column. - parallel_workers: - Worker count for independent diagnostic tasks. - dataframe: - Optional in-memory DataFrame. When supplied it takes precedence over - ``file_path`` and is copied before analysis. - write_artifacts: - Persist cleaned CSV/chart artifacts. Web application callers keep this - enabled; reusable library calls can disable filesystem side effects. - """ + """Run the complete FrameVitals analysis pipeline.""" overall_start = time.perf_counter() timings_ms: dict[str, Any] = {} + disabled = set(disabled_modules or ()) + unknown_modules = sorted(disabled - VALID_MODULES) + if unknown_modules: + raise ValueError( + "Unknown disabled module(s): " + ", ".join(unknown_modules) + ) + module_status: dict[str, str] = { + name: "pending" for name in sorted(VALID_MODULES) + } + + def module_enabled(name: str) -> bool: + return name not in disabled + # Phase 1: load and understand structure. t0 = time.perf_counter() if dataframe is not None: @@ -109,6 +120,12 @@ def run_full_analysis( if target_column is not None and target_column not in df.columns: raise ValueError(f"Target column not found: {target_column}") + execution_budget = derive_execution_budget( + len(df), + len(df.columns), + mode=analysis_mode, + ) + t0 = time.perf_counter() profile = build_profile(df) timings_ms["profile"] = (time.perf_counter() - t0) * 1000 @@ -132,9 +149,14 @@ def run_full_analysis( analysis_mode=analysis_mode, target_column=target_column, ) + analysis_selection["execution_modules"] = { + "disabled": sorted(disabled), + "enabled": sorted(VALID_MODULES - disabled), + } + analysis_selection["execution_budget"] = execution_budget.to_dict() timings_ms["analysis_selection"] = (time.perf_counter() - t0) * 1000 - # Phase 2: quality scoring. + # Phase 2: core quality/readiness plus bounded practical diagnostics. t0 = time.perf_counter() health = calculate_health_score(df, profile) timings_ms["health"] = (time.perf_counter() - t0) * 1000 @@ -147,108 +169,247 @@ def run_full_analysis( advanced = calculate_advanced_indicators(df) timings_ms["advanced"] = (time.perf_counter() - t0) * 1000 - # Phase 3: independent heavier analyses. + if module_enabled("quality_diagnostics"): + _, quality_diagnostics, quality_elapsed = _safe_call( + "quality_diagnostics", + lambda: run_quality_diagnostics( + df, + profile=profile, + column_roles=column_roles, + max_sample_rows=max(execution_budget.quality_sample_rows, 10), + ), + ) + timings_ms["quality_diagnostics"] = quality_elapsed + module_status["quality_diagnostics"] = _result_status(quality_diagnostics) + else: + quality_diagnostics = _skipped_module("quality_diagnostics") + timings_ms["quality_diagnostics"] = 0.0 + module_status["quality_diagnostics"] = "disabled" + + # Phase 3: independent heavier analyses. Legacy algorithms are routed + # through bounded adapters until their streaming/native replacements land. deep_statistics_v2 = None anomalies_v2 = None time_series_analysis = None text_profile = None - if analysis_mode in {"standard", "deep", "research"}: - tasks: list[tuple[str, Callable[[], Any]]] = [ - ("deep_statistics_v2", lambda: run_deep_statistics_v2(df)), - ("anomalies_v2", lambda: detect_anomalies_ensemble(df)), - ( - "time_series", - lambda: detect_and_analyze_time_series( - df, - target_column=target_column, - ), + phase3_modules: list[tuple[str, str, Callable[[], Any]]] = [ + ( + "deep_statistics", + "deep_statistics_v2", + lambda: run_budgeted_deep_statistics(df, budget=execution_budget), + ), + ( + "anomaly_detection", + "anomalies_v2", + lambda: run_budgeted_anomalies(df, budget=execution_budget), + ), + ( + "time_series", + "time_series", + lambda: run_budgeted_time_series( + df, + budget=execution_budget, + target_column=target_column, ), - ("text_profile", lambda: profile_text_columns(df)), - ] + ), + ("text_profile", "text_profile", lambda: profile_text_columns(df)), + ] + + phase3_results: dict[str, Any] = {} + phase3_worker_limit = min( + parallel_workers, + execution_budget.max_memory_heavy_parallelism, + ) + if analysis_mode in {"standard", "deep", "research"}: + tasks: list[tuple[str, str, Callable[[], Any]]] = [] + for module, result_key, fn in phase3_modules: + if module_enabled(module): + tasks.append((module, result_key, fn)) + module_status[module] = "scheduled" + else: + phase3_results[result_key] = _skipped_module(module) + module_status[module] = "disabled" - results: dict[str, Any] = {} per_task_ms: dict[str, float] = {} - - phase3_start = time.perf_counter() - with ThreadPoolExecutor(max_workers=parallel_workers) as executor: - futures = { - executor.submit(_safe_call, name, fn): name for name, fn in tasks - } - for future in as_completed(futures): - name, value, elapsed = future.result() - results[name] = value - per_task_ms[name] = elapsed - - deep_statistics_v2 = results.get("deep_statistics_v2") - anomalies_v2 = results.get("anomalies_v2") - time_series_analysis = results.get("time_series") - text_profile = results.get("text_profile") - - timings_ms["phase3_parallel_total"] = ( - time.perf_counter() - phase3_start - ) * 1000 + if tasks: + phase3_start = time.perf_counter() + with ThreadPoolExecutor(max_workers=phase3_worker_limit) as executor: + futures = { + executor.submit(_safe_call, result_key, fn): (module, result_key) + for module, result_key, fn in tasks + } + for future in as_completed(futures): + module, result_key = futures[future] + name, value, elapsed = future.result() + phase3_results[result_key] = value + per_task_ms[name] = elapsed + module_status[module] = _result_status(value) + + timings_ms["phase3_parallel_total"] = ( + time.perf_counter() - phase3_start + ) * 1000 + else: + timings_ms["phase3_parallel_total"] = 0.0 timings_ms["phase3_tasks"] = per_task_ms + else: + for module, _, _ in phase3_modules: + module_status[module] = "not_applicable" + + deep_statistics_v2 = phase3_results.get("deep_statistics_v2") + anomalies_v2 = phase3_results.get("anomalies_v2") + time_series_analysis = phase3_results.get("time_series") + text_profile = phase3_results.get("text_profile") - # Phase 4: target-aware ML chain. + # Phase 4: target-aware diagnostics and ML chain. + target_intelligence = None model_leaderboard = None explainability = None - if target_column and analysis_mode in {"standard", "deep", "research"}: - t0 = time.perf_counter() - _, model_leaderboard, _ = _safe_call( - "model_leaderboard", - lambda: run_model_leaderboard(df, target_column=target_column), - ) - timings_ms["model_leaderboard"] = (time.perf_counter() - t0) * 1000 - if ( - isinstance(model_leaderboard, dict) - and model_leaderboard.get("available") - and model_leaderboard.get("winner") - ): + if target_column: + if module_enabled("target_intelligence"): t0 = time.perf_counter() - _, explainability, _ = _safe_call( - "explainability", - lambda: explain_winner( + _, target_intelligence, _ = _safe_call( + "target_intelligence", + lambda: run_target_intelligence( + df, + target_column=target_column, + column_roles=column_roles, + ), + ) + timings_ms["target_intelligence"] = (time.perf_counter() - t0) * 1000 + module_status["target_intelligence"] = _result_status(target_intelligence) + else: + target_intelligence = _skipped_module("target_intelligence") + timings_ms["target_intelligence"] = 0.0 + module_status["target_intelligence"] = "disabled" + else: + module_status["target_intelligence"] = "not_applicable" + + modeling_applicable = target_column and analysis_mode in {"standard", "deep", "research"} + if modeling_applicable: + if module_enabled("modeling"): + t0 = time.perf_counter() + _, model_leaderboard, _ = _safe_call( + "model_leaderboard", + lambda: run_model_leaderboard(df, target_column=target_column), + ) + timings_ms["model_leaderboard"] = (time.perf_counter() - t0) * 1000 + module_status["modeling"] = _result_status(model_leaderboard) + else: + model_leaderboard = _skipped_module("modeling") + timings_ms["model_leaderboard"] = 0.0 + module_status["modeling"] = "disabled" + else: + module_status["modeling"] = "not_applicable" + + winner_available = ( + isinstance(model_leaderboard, dict) + and model_leaderboard.get("available") + and model_leaderboard.get("winner") + ) + if winner_available: + if module_enabled("explainability"): + t0 = time.perf_counter() + + def run_explainability(): + from framevitals.explainability import explain_winner + + return explain_winner( df, target_column=target_column, leaderboard_result=model_leaderboard, dataset_id=dataset_id, - ), + ) + + _, explainability, _ = _safe_call( + "explainability", + run_explainability, ) timings_ms["explainability"] = (time.perf_counter() - t0) * 1000 + module_status["explainability"] = _result_status(explainability) + else: + explainability = _skipped_module("explainability") + timings_ms["explainability"] = 0.0 + module_status["explainability"] = "disabled" + elif not module_enabled("explainability"): + explainability = _skipped_module("explainability") + module_status["explainability"] = "disabled" + else: + module_status["explainability"] = "not_applicable" # Phase 5: signals, cleaning, and optional visual artifacts. t0 = time.perf_counter() signals = build_signals(profile, health, ml_readiness, advanced) timings_ms["signals"] = (time.perf_counter() - t0) * 1000 - t0 = time.perf_counter() - cleaning = create_cleaned_dataset( - dataset_id, - df, - write_output=write_artifacts, - before_profile=profile, - before_health=health, - ) - timings_ms["cleaning"] = (time.perf_counter() - t0) * 1000 - - charts: list[dict] = [] - if write_artifacts and analysis_mode in {"standard", "deep", "research"}: + if module_enabled("cleaning"): t0 = time.perf_counter() - charts = generate_charts( + cleaning = create_cleaned_dataset( dataset_id, df, - health, - advanced, - cleaning, - target_column=target_column, - model_leaderboard=model_leaderboard, - explainability=explainability, - time_series=time_series_analysis, - deep_statistics_v2=deep_statistics_v2, + write_output=write_artifacts, + before_profile=profile, + before_health=health, ) - timings_ms["charts"] = (time.perf_counter() - t0) * 1000 + timings_ms["cleaning"] = (time.perf_counter() - t0) * 1000 + module_status["cleaning"] = "ran" + else: + missing_count = sum( + int(value) + for value in profile.get("missing_counts", {}).values() + if value is not None + ) + duplicate_count = int(profile.get("duplicate_rows", 0) or 0) + cleaning = { + **_skipped_module("cleaning"), + "actions": [], + "before_health": health, + "after_health": health, + "output_path": None, + "missing_before": missing_count, + "missing_after": missing_count, + "duplicates_before": duplicate_count, + "duplicates_after": duplicate_count, + } + timings_ms["cleaning"] = 0.0 + module_status["cleaning"] = "disabled" + + charts: list[dict] = [] + charts_applicable = write_artifacts and analysis_mode in {"standard", "deep", "research"} + if charts_applicable and module_enabled("charts"): + t0 = time.perf_counter() + + def render_charts(): + from framevitals.visualizer import generate_charts + + return generate_charts( + dataset_id, + df, + health, + advanced, + cleaning, + target_column=target_column, + model_leaderboard=model_leaderboard, + explainability=explainability, + time_series=time_series_analysis, + deep_statistics_v2=deep_statistics_v2, + ) + + _, rendered_charts, chart_elapsed = _safe_call("charts", render_charts) + timings_ms["charts"] = chart_elapsed + if isinstance(rendered_charts, list): + charts = rendered_charts + module_status["charts"] = "ran" + else: + charts = [] + module_status["charts"] = _result_status(rendered_charts) + elif not module_enabled("charts"): + timings_ms["charts"] = 0.0 + module_status["charts"] = "disabled" + else: + timings_ms["charts"] = 0.0 + module_status["charts"] = "not_applicable" # Phase 6: optional AI interpretation. ai_env = os.environ.get( @@ -257,13 +418,22 @@ def run_full_analysis( ) analyze_ai_default = ai_env.strip().lower() in {"1", "true", "yes"} - if skip_ai or not analyze_ai_default: + if not module_enabled("ai"): + ai_report = { + "source": "disabled", + "text": "AI report disabled by configuration.", + "deferred": False, + } + timings_ms["ai_report"] = 0.0 + module_status["ai"] = "disabled" + elif skip_ai or not analyze_ai_default: ai_report = { "source": "deferred" if not skip_ai else "skipped", "text": "" if not skip_ai else "AI report skipped.", "deferred": not skip_ai, } timings_ms["ai_report"] = 0.0 + module_status["ai"] = "deferred" if not skip_ai else "skipped_by_caller" else: t0 = time.perf_counter() try: @@ -276,9 +446,11 @@ def run_full_analysis( column_roles_summary=roles_summary, dataset_signals=dataset_signals, ) + module_status["ai"] = "ran" except Exception as exc: # noqa: BLE001 logger.exception("AI report generation failed") ai_report = {"source": f"error: {exc}", "text": str(exc)} + module_status["ai"] = "error" timings_ms["ai_report"] = (time.perf_counter() - t0) * 1000 timings_ms["total"] = (time.perf_counter() - overall_start) * 1000 @@ -296,6 +468,12 @@ def run_full_analysis( "filename": original_filename, "analysis_mode": analysis_mode, "artifacts_enabled": write_artifacts, + "execution": { + "disabled_modules": sorted(disabled), + "module_status": module_status, + "budget": execution_budget.to_dict(), + "phase3_worker_limit": int(phase3_worker_limit), + }, "profile": profile, "column_roles": column_roles, "roles_summary": roles_summary, @@ -305,8 +483,10 @@ def run_full_analysis( "signals": signals, "ml_readiness": ml_readiness, "advanced": advanced, + "quality_diagnostics": quality_diagnostics, "deep_statistics_v2": deep_statistics_v2, "anomalies_v2": anomalies_v2, + "target_intelligence": target_intelligence, "model_leaderboard": model_leaderboard, "explainability": explainability, "time_series": time_series_analysis, diff --git a/src/framevitals/planning.py b/src/framevitals/planning.py new file mode 100644 index 0000000..43d0e61 --- /dev/null +++ b/src/framevitals/planning.py @@ -0,0 +1,88 @@ +"""Analysis planning result objects and human-readable explanations.""" + +from __future__ import annotations + +from typing import Any + + +class AnalysisPlan(dict): + """Dict-compatible preview of analyses FrameVitals considers applicable.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + @property + def selection(self) -> dict[str, Any]: + value = self.get("selection", {}) + return value if isinstance(value, dict) else {} + + @property + def selected(self) -> list[dict[str, Any]]: + value = self.selection.get("selected_analyses", []) + return value if isinstance(value, list) else [] + + @property + def skipped(self) -> list[dict[str, Any]]: + value = self.selection.get("skipped_analyses", []) + return value if isinstance(value, list) else [] + + @property + def recommended(self) -> list[dict[str, Any]]: + value = self.selection.get("recommended_analyses", []) + return value if isinstance(value, list) else [] + + def summary(self) -> dict[str, Any]: + return { + "dataset_name": self.get("dataset_name"), + "analysis_mode": self.get("analysis_mode"), + "target": self.get("target"), + "shape": self.get("shape", {}), + "selected_count": len(self.selected), + "skipped_count": len(self.skipped), + "recommended_count": len(self.recommended), + } + + def explain_text(self) -> str: + """Render a terminal-friendly explanation of the current plan.""" + shape = self.get("shape", {}) or {} + lines = [ + "FrameVitals Analysis Plan", + "=" * 72, + f"Dataset {self.get('dataset_name', '')}", + f"Mode {self.get('analysis_mode', 'unknown')}", + f"Target {self.get('target') or ''}", + f"Shape {shape.get('rows', '?')} rows x {shape.get('columns', '?')} columns", + "", + f"Selected {len(self.selected)}", + ] + + for item in self.selected: + lines.append( + f" [RUN] {item.get('id', ''):<28} {item.get('name', '')}" + ) + + lines.extend(["", f"Recommended {len(self.recommended)}"]) + for item in self.recommended: + lines.append( + f" [NEXT] {item.get('id', ''):<28} {item.get('reason', '')}" + ) + + lines.extend(["", f"Skipped {len(self.skipped)}"]) + for item in self.skipped[:12]: + reason = " ".join(str(item.get("reason", "")).split()) + if len(reason) > 68: + reason = reason[:67].rstrip() + "…" + lines.append( + f" [SKIP] {item.get('id', ''):<28} {reason}" + ) + if len(self.skipped) > 12: + lines.append(f" ... and {len(self.skipped) - 12} more skipped analyses") + + lines.extend([ + "=" * 72, + "This is a preview only; no heavy model/statistics stage was executed.", + ]) + return "\n".join(lines) diff --git a/src/framevitals/planning_api.py b/src/framevitals/planning_api.py new file mode 100644 index 0000000..d3063ed --- /dev/null +++ b/src/framevitals/planning_api.py @@ -0,0 +1,273 @@ +"""Planning-only public execution path. + +`fv.plan()` should be cheap enough to call before committing to a full analysis. +This module deliberately avoids importing the full FrameVitals pipeline while +still resolving configuration, structural signals, and adaptive execution +budgets exactly as the execution layer expects. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +import pandas as pd + +from framevitals.analysis_selector import select_analyses +from framevitals.column_roles import infer_column_roles +from framevitals.config import ConfigInput, VALID_MODULES, resolve_config +from framevitals.dataset_signals import detect_dataset_signals +from framevitals.execution import ( + derive_execution_budget, + derive_streaming_profile_column_limit, +) +from framevitals.planning import AnalysisPlan +from framevitals.profiler import build_profile +from framevitals.sources import StreamingDatasetSource, resolve_source + + +DataInput = str | Path | pd.DataFrame +PLANNING_SAMPLE_ROWS = 5_000 + + +def _evenly_spaced_names(names: Sequence[str], limit: int) -> list[str]: + """Select a deterministic schema-wide column projection without NumPy.""" + total = len(names) + if limit < 1: + raise ValueError("column projection limit must be at least 1") + if total <= limit: + return list(names) + if limit == 1: + return [str(names[0])] + + selected = [ + str(names[(index * (total - 1)) // (limit - 1)]) + for index in range(limit) + ] + return list(dict.fromkeys(selected)) + + +def _streaming_projection( + source: StreamingDatasetSource, + *, + source_columns: int, + limit: int, + target: str | None, +) -> list[str] | None: + """Resolve a bounded projection for ultra-wide streaming planning.""" + if source_columns <= limit: + return None + + schema_method = getattr(source, "schema", None) + if not callable(schema_method): + raise TypeError( + "Ultra-wide streaming planning requires source.schema() so FrameVitals " + "can project columns instead of materializing the complete width." + ) + schema = schema_method() + names = [str(field.name) for field in schema] + if target is not None and target not in names: + raise ValueError(f"Target column not found: {target}") + + selected = _evenly_spaced_names(names, limit) + if target is not None and target not in selected: + if selected: + selected[-1] = target + else: + selected = [target] + return list(dict.fromkeys(selected)) + + +def _streaming_head_sample( + source: StreamingDatasetSource, + *, + max_rows: int = PLANNING_SAMPLE_ROWS, + columns: Sequence[str] | None = None, +) -> pd.DataFrame: + """Read at most ``max_rows`` rows and stop; planning must not scan the file.""" + frames: list[pd.DataFrame] = [] + remaining = int(max_rows) + for batch in source.iter_batches(batch_size=max_rows, columns=columns): + if remaining <= 0: + break + take = min(remaining, int(batch.num_rows)) + if take > 0: + frames.append(batch.slice(0, take).to_pandas()) + remaining -= take + if remaining <= 0: + break + if not frames: + raise ValueError("Streaming dataset produced no rows for planning.") + return pd.concat(frames, ignore_index=True) + + +def _project_sample_profile_to_source( + profile: dict[str, Any], + *, + source_rows: int, + source_columns: int, + sample_rows: int, + profiled_columns: int, +) -> dict[str, Any]: + """Scale rate-based sample metrics to the true source shape for planning.""" + projected = dict(profile) + factor = source_rows / max(sample_rows, 1) + projected["shape"] = {"rows": int(source_rows), "columns": int(source_columns)} + projected["missing_counts"] = { + column: int(round(float(value or 0) * factor)) + for column, value in profile.get("missing_counts", {}).items() + } + sample_duplicate_rows = int(profile.get("duplicate_rows", 0) or 0) + projected["duplicate_rows"] = int(round(sample_duplicate_rows * factor)) + projected["planning_sample_metadata"] = { + "sampled": sample_rows < source_rows, + "sample_rows": int(sample_rows), + "source_rows": int(source_rows), + "strategy": "bounded_head", + "full_scan": False, + "rate_metrics_projected": sample_rows < source_rows, + "profiled_columns": int(profiled_columns), + "source_columns": int(source_columns), + "column_sampled": int(profiled_columns) < int(source_columns), + } + return projected + + +def plan( + data: DataInput, + *, + target: str | None = None, + mode: str | None = None, + workers: int | None = None, + preset: str | None = None, + config: ConfigInput = None, + disabled_modules: list[str] | tuple[str, ...] | None = None, +) -> AnalysisPlan: + """Preview planned analyses, scale policy, and execution constraints.""" + resolved = resolve_config( + config, + preset=preset, + mode=mode, + target=target, + workers=workers, + artifacts=False, + disabled_modules=disabled_modules, + ) + + source = resolve_source(data) + source_metadata = source.inspect() + + if source_metadata.supports_streaming and isinstance(source, StreamingDatasetSource): + source_rows = int(source_metadata.rows or 0) + source_columns = int(source_metadata.columns or 0) + if source_rows < 1 or source_columns < 1: + raise ValueError(f"Dataset is empty or has no columns: {source_metadata.name}") + + column_limit = derive_streaming_profile_column_limit( + source_rows, + source_columns, + mode=resolved.mode, + ) + projected_columns = _streaming_projection( + source, + source_columns=source_columns, + limit=column_limit, + target=resolved.target, + ) + dataframe = _streaming_head_sample( + source, + columns=projected_columns, + ) + profiled_columns = int(len(dataframe.columns)) + dataset_profile = _project_sample_profile_to_source( + build_profile(dataframe), + source_rows=source_rows, + source_columns=source_columns, + sample_rows=len(dataframe), + profiled_columns=profiled_columns, + ) + planning_data = { + "materialized_full_dataset": False, + "sampled": len(dataframe) < source_rows, + "sample_rows": int(len(dataframe)), + "source_rows": source_rows, + "strategy": "bounded_head", + "full_scan": False, + "source_columns": source_columns, + "sample_columns": profiled_columns, + "column_sampled": profiled_columns < source_columns, + "column_strategy": ( + "deterministic_schema_projection" + if profiled_columns < source_columns + else "full_schema" + ), + "column_limit": int(column_limit), + } + else: + dataframe = source.load() + source_rows = int(len(dataframe)) + source_columns = int(len(dataframe.columns)) + dataset_profile = build_profile(dataframe) + planning_data = { + "materialized_full_dataset": True, + "sampled": False, + "sample_rows": source_rows, + "source_rows": source_rows, + "strategy": "full_dataset", + "full_scan": True, + "source_columns": source_columns, + "sample_columns": source_columns, + "column_sampled": False, + "column_strategy": "full_schema", + "column_limit": source_columns, + } + + source_columns_list = list(dataset_profile.get("columns", dataframe.columns)) + if resolved.target is not None and resolved.target not in source_columns_list: + raise ValueError(f"Target column not found: {resolved.target}") + + column_roles = infer_column_roles(dataframe) + dataset_signals = detect_dataset_signals( + dataframe, + dataset_profile, + column_roles=column_roles, + source_shape=(source_rows, source_columns), + ) + + budget = derive_execution_budget( + source_rows, + source_columns, + mode=resolved.mode, + ) + + selection = select_analyses( + signals=dataset_signals, + analysis_mode=resolved.mode, + target_column=resolved.target, + ) + disabled = set(resolved.disabled_modules) + selection["execution_modules"] = { + "disabled": sorted(disabled), + "enabled": sorted(VALID_MODULES - disabled), + } + selection["execution_budget"] = budget.to_dict() + + public_signals = { + key: value + for key, value in dataset_signals.items() + if key != "column_roles" + } + + return AnalysisPlan({ + "dataset_name": source_metadata.name, + "source": source_metadata.to_dict(), + "planning_data": planning_data, + "analysis_mode": resolved.mode, + "target": resolved.target, + "shape": dict(dataset_profile.get("shape", {})), + "config": resolved.to_dict(), + "execution_budget": budget.to_dict(), + "signals": public_signals, + "selection": selection, + }) diff --git a/src/framevitals/plugins.py b/src/framevitals/plugins.py new file mode 100644 index 0000000..b9ce2a2 --- /dev/null +++ b/src/framevitals/plugins.py @@ -0,0 +1,86 @@ +"""Opt-in discovery for third-party FrameVitals extensions. + +FrameVitals never imports installed plugins automatically. Applications that +want extension discovery must call :func:`discover_checks` explicitly, then +pass the returned definitions to ``framevitals.run_checks`` or +``framevitals.gate``. + +Third-party packages register one check per Python entry point under the +``framevitals.checks`` group:: + + [project.entry-points."framevitals.checks"] + positive_revenue = "acme_data_checks:positive_revenue" + +The referenced object must be either a :class:`framevitals.checks.DataCheck` +or a callable accepting one pandas DataFrame. +""" + +from __future__ import annotations + +from importlib import metadata as importlib_metadata +from typing import Any + +from framevitals.checks import DataCheck + + +CHECK_ENTRYPOINT_GROUP = "framevitals.checks" + + +def _entry_points_for(group: str): + discovered = importlib_metadata.entry_points() + if hasattr(discovered, "select"): + return list(discovered.select(group=group)) + # Compatibility with older importlib.metadata collection objects. + return list(discovered.get(group, ())) + + +def _load_entry_point(entry: Any) -> DataCheck: + try: + loaded = entry.load() + except Exception as exc: + raise RuntimeError( + f"Failed to load FrameVitals check plugin {entry.name!r} from {entry.value!r}." + ) from exc + + if isinstance(loaded, DataCheck): + return loaded + if callable(loaded): + return DataCheck(name=str(entry.name), function=loaded) + raise TypeError( + "FrameVitals check plugin " + f"{entry.name!r} must expose a DataCheck or DataFrame callable; " + f"got {type(loaded).__name__}." + ) + + +def discover_checks( + *, + group: str = CHECK_ENTRYPOINT_GROUP, +) -> list[DataCheck]: + """Load explicitly installed check plugins from a Python entry-point group. + + Discovery is opt-in because loading an entry point executes code from the + installed provider package. Returned checks are sorted by registration + name for deterministic behavior. Duplicate public check names are rejected + so gate behavior cannot depend on package discovery order. + """ + entries = sorted( + _entry_points_for(group), + key=lambda entry: (str(entry.name), str(entry.value)), + ) + checks: list[DataCheck] = [] + seen_names: set[str] = set() + + for entry in entries: + definition = _load_entry_point(entry) + if definition.name in seen_names: + raise ValueError( + f"Duplicate FrameVitals check plugin name: {definition.name!r}." + ) + seen_names.add(definition.name) + checks.append(definition) + + return checks + + +__all__ = ["CHECK_ENTRYPOINT_GROUP", "discover_checks"] diff --git a/src/framevitals/profiler.py b/src/framevitals/profiler.py index 9847edb..31c9160 100644 --- a/src/framevitals/profiler.py +++ b/src/framevitals/profiler.py @@ -1,6 +1,19 @@ +from __future__ import annotations + +import os + import numpy as np import pandas as pd +from framevitals.backends import numeric_profile, resolve_numeric_backend +from framevitals.execution import _deterministic_stratified_positions + +MAX_CORRELATION_COLUMNS = 100 +MAX_EXACT_DUPLICATE_CELLS = 50_000_000 +DUPLICATE_SAMPLE_ROWS = 50_000 +NATIVE_NUMERIC_PROFILE_MIN_ROWS = 50_000 +NATIVE_NUMERIC_PROFILE_MIN_CELLS = 1_000_000 + def clean_value(value): if pd.isna(value): @@ -36,24 +49,209 @@ def detect_column_types(df): return numeric_cols, categorical_cols, date_cols +def _missing_counts( + df: pd.DataFrame, + *, + precomputed: dict[str, int] | None = None, +) -> pd.Series: + """Count missing values without materializing a frame-sized boolean table.""" + known = precomputed or {} + return pd.Series( + { + column: int(known[column]) if column in known else int(df[column].isna().sum()) + for column in df.columns + }, + dtype="int64", + ) + + +def _duplicate_profile(df: pd.DataFrame) -> tuple[int, dict]: + """Return duplicate estimate plus explicit execution metadata.""" + rows, columns = df.shape + cells = rows * columns + if cells <= MAX_EXACT_DUPLICATE_CELLS: + count = int(df.duplicated().sum()) + return count, { + "method": "exact", + "sampled": False, + "sample_rows": rows, + } + + sample_rows = min(rows, DUPLICATE_SAMPLE_ROWS) + positions = _deterministic_stratified_positions(rows, sample_rows) + sample = df.iloc[positions] + sample_duplicates = int(sample.duplicated().sum()) + rate = sample_duplicates / max(len(sample), 1) + estimate = int(round(rate * rows)) + return estimate, { + "method": "sample_estimate", + "sampled": True, + "sample_rows": int(len(sample)), + "source_rows": int(rows), + "estimated_duplicate_rate": round(float(rate), 6), + "strategy": "stratified_jitter_global_rows", + } + + +def _bounded_correlations( + df: pd.DataFrame, + numeric_cols: list[str], +) -> tuple[dict, dict]: + if len(numeric_cols) < 2: + return {}, { + "method": "not_applicable", + "columns_used": len(numeric_cols), + "columns_available": len(numeric_cols), + "truncated": False, + } + + selected = numeric_cols + truncated = len(selected) > MAX_CORRELATION_COLUMNS + if truncated: + non_missing = { + column: int(df[column].notna().sum()) + for column in numeric_cols + } + selected = sorted( + numeric_cols, + key=lambda column: (-non_missing[column], str(column)), + )[:MAX_CORRELATION_COLUMNS] + + finite_numeric = df[selected].replace([np.inf, -np.inf], np.nan) + correlations = ( + finite_numeric + .corr(numeric_only=True) + .round(3) + .replace({np.nan: None}) + .to_dict() + ) + return correlations, { + "method": "dense_bounded" if truncated else "dense_exact_columns", + "columns_used": len(selected), + "columns_available": len(numeric_cols), + "truncated": truncated, + "max_columns": MAX_CORRELATION_COLUMNS, + } + + +def _round_optional(value, digits: int = 3): + if value is None: + return None + return round(float(value), digits) + + +def _pandas_numeric_summary( + df: pd.DataFrame, + numeric_cols: list[str], +) -> tuple[dict, dict, dict[str, int]]: + if not numeric_cols: + return {}, { + "backend": "not_applicable", + "method": "not_applicable", + "approximate_quantiles": False, + }, {} + + finite_numeric = df[numeric_cols].replace([np.inf, -np.inf], np.nan) + summary = ( + finite_numeric + .describe() + .T + .replace({np.nan: None}) + .round(3) + .to_dict(orient="index") + ) + return summary, { + "backend": "pandas", + "method": "pandas_describe", + "approximate_quantiles": False, + "finite_only_moments": True, + "columns_profiled": len(numeric_cols), + }, {} + + +def _native_numeric_summary( + df: pd.DataFrame, + numeric_cols: list[str], +) -> tuple[dict, dict, dict[str, int]]: + summary: dict[str, dict] = {} + missing: dict[str, int] = {} + relative_accuracy = None + + for stream_id, column in enumerate(numeric_cols): + payload = numeric_profile(df[column], backend="rust", stream_id=stream_id) + quantiles = payload.get("quantiles", {}) + relative_accuracy = quantiles.get("relative_accuracy", relative_accuracy) + missing[column] = int(payload["missing"]) + summary[column] = { + "count": int(payload["count"]), + "mean": _round_optional(payload.get("mean")), + "std": _round_optional(payload.get("std")), + "min": _round_optional(payload.get("minimum")), + "25%": _round_optional(quantiles.get("p25")), + "50%": _round_optional(quantiles.get("p50")), + "75%": _round_optional(quantiles.get("p75")), + "max": _round_optional(payload.get("maximum")), + } + + return summary, { + "backend": "rust", + "method": "native_fused_numeric_column_scan", + "approximate_quantiles": True, + "quantile_relative_accuracy": relative_accuracy, + "finite_only_moments": True, + "columns_profiled": len(numeric_cols), + "raw_observations_retained": False, + }, missing + + +def _numeric_summary( + df: pd.DataFrame, + numeric_cols: list[str], +) -> tuple[dict, dict, dict[str, int]]: + if not numeric_cols: + return _pandas_numeric_summary(df, numeric_cols) + + requested = os.getenv("FRAMEVITALS_BACKEND", "auto").strip().lower() + selected = resolve_numeric_backend() + numeric_cells = int(len(df)) * len(numeric_cols) + native_worthwhile = ( + len(df) >= NATIVE_NUMERIC_PROFILE_MIN_ROWS + or numeric_cells >= NATIVE_NUMERIC_PROFILE_MIN_CELLS + ) + use_native = selected == "rust" and (requested == "rust" or native_worthwhile) + + if not use_native: + summary, metadata, missing = _pandas_numeric_summary(df, numeric_cols) + metadata["native_eligible"] = selected == "rust" + metadata["native_threshold_reached"] = native_worthwhile + return summary, metadata, missing + + try: + return _native_numeric_summary(df, numeric_cols) + except Exception as exc: + if requested == "rust": + raise + summary, metadata, missing = _pandas_numeric_summary(df, numeric_cols) + metadata.update({ + "native_eligible": True, + "native_threshold_reached": True, + "fallback_from": "rust", + "fallback_reason": f"{type(exc).__name__}: {exc}", + }) + return summary, metadata, missing + + def build_profile(df): rows, columns = df.shape numeric_cols, categorical_cols, date_cols = detect_column_types(df) - missing_counts = df.isna().sum() + numeric_summary, numeric_summary_metadata, numeric_missing = _numeric_summary( + df, + numeric_cols, + ) + missing_counts = _missing_counts(df, precomputed=numeric_missing) missing_percent = (missing_counts / max(rows, 1) * 100).round(2) - duplicate_rows = int(df.duplicated().sum()) - - numeric_summary = {} - if numeric_cols: - numeric_summary = ( - df[numeric_cols] - .describe() - .T - .replace({np.nan: None}) - .round(3) - .to_dict(orient="index") - ) + duplicate_rows, duplicate_metadata = _duplicate_profile(df) categorical_summary = {} for col in categorical_cols: @@ -63,17 +261,10 @@ def build_profile(df): "top_values": {str(k): int(v) for k, v in counts.items()}, } - correlations = {} - if len(numeric_cols) >= 2: - correlations = ( - df[numeric_cols] - .corr(numeric_only=True) - .round(3) - .replace({np.nan: None}) - .to_dict() - ) + correlations, correlation_metadata = _bounded_correlations(df, numeric_cols) - preview = df.head(15).where(df.notna(), None).to_dict(orient="records") + preview_frame = df.head(15) + preview = preview_frame.where(preview_frame.notna(), None).to_dict(orient="records") return { "shape": {"rows": rows, "columns": columns}, @@ -86,9 +277,12 @@ def build_profile(df): "missing_percent": series_to_dict(missing_percent), "duplicate_rows": duplicate_rows, "duplicate_percent": round(duplicate_rows / max(rows, 1) * 100, 2), + "duplicate_metadata": duplicate_metadata, "memory_usage_mb": round(df.memory_usage(deep=True).sum() / (1024 * 1024), 3), "numeric_summary": numeric_summary, + "numeric_summary_metadata": numeric_summary_metadata, "categorical_summary": categorical_summary, "correlations": correlations, + "correlation_metadata": correlation_metadata, "preview": preview, } diff --git a/src/framevitals/provenance.py b/src/framevitals/provenance.py new file mode 100644 index 0000000..16db8c9 --- /dev/null +++ b/src/framevitals/provenance.py @@ -0,0 +1,102 @@ +"""Shared execution-provenance helpers for public FrameVitals results. + +The 0.x API historically grew execution metadata inside individual diagnostics. +This module defines a small additive contract so those payloads can converge +without breaking callers that still rely on legacy fields. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + + +EXECUTION_SCHEMA_VERSION = "1" + + +def load_fully_materializes(metadata: Any) -> bool: + """Return whether ``source.load()`` creates a complete pandas representation. + + A pandas DataFrame is already materialized before FrameVitals sees it. File, + Arrow, relation, remote, and other source types may all require a complete + conversion when an exact operation calls ``load()``; source kind alone is + therefore not a reliable materialization signal. + """ + return not ( + getattr(metadata, "kind", None) == "memory" + and getattr(metadata, "format", None) == "pandas" + ) + + +def execution_provenance( + method: str, + *, + full_materialization: bool, + source: Mapping[str, Any] | None = None, + sampled: bool | None = None, + source_rows: int | None = None, + source_columns: int | None = None, + sample_rows: int | None = None, + strategy: str | None = None, + components: Mapping[str, Any] | None = None, + reason: str | None = None, + scope: str | None = None, + extra: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build a JSON-friendly execution block using the shared v1 vocabulary. + + Optional values are omitted rather than serialized as ``null`` so existing + result payloads can adopt the schema incrementally. ``extra`` exists for + operation-specific fields such as scale class, pair budget, or projection + counts without making them part of the universal contract. + """ + if not str(method).strip(): + raise ValueError("execution provenance method must not be empty.") + + payload: dict[str, Any] = { + "execution_schema_version": EXECUTION_SCHEMA_VERSION, + "method": str(method), + "full_materialization": bool(full_materialization), + } + optional = { + "source": dict(source) if source is not None else None, + "sampled": bool(sampled) if sampled is not None else None, + "source_rows": int(source_rows) if source_rows is not None else None, + "source_columns": int(source_columns) if source_columns is not None else None, + "sample_rows": int(sample_rows) if sample_rows is not None else None, + "strategy": strategy, + "components": dict(components) if components is not None else None, + "reason": reason, + "scope": scope, + } + payload.update({key: value for key, value in optional.items() if value is not None}) + if extra: + payload.update(dict(extra)) + return payload + + +def normalize_execution( + execution: Mapping[str, Any], + *, + method: str | None = None, + full_materialization: bool | None = None, + source: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Add the shared v1 contract to an existing legacy execution mapping. + + Existing keys always win unless the caller explicitly supplies a method, + materialization flag, or source. This makes migration additive during 0.x. + """ + result = dict(execution) + result["execution_schema_version"] = EXECUTION_SCHEMA_VERSION + + resolved_method = method or result.get("method") or result.get("scope") + if resolved_method is not None: + result["method"] = str(resolved_method) + if full_materialization is not None: + result["full_materialization"] = bool(full_materialization) + else: + result.setdefault("full_materialization", False) + if source is not None: + result["source"] = dict(source) + return result diff --git a/src/framevitals/py.typed b/src/framevitals/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/framevitals/quality_diagnostics.py b/src/framevitals/quality_diagnostics.py new file mode 100644 index 0000000..29ded4a --- /dev/null +++ b/src/framevitals/quality_diagnostics.py @@ -0,0 +1,537 @@ +"""Deterministic data-quality diagnostics for tabular datasets. + +The diagnostics in this module are intentionally explainable and resource +bounded. Existing profile/role metadata is reused where possible. Potentially +expensive value-level checks operate on a deterministic bounded sample, while +exact full-column comparisons are reserved for duplicate-column candidates +identified by a sample fingerprint first. +""" + +from __future__ import annotations + +import hashlib +from itertools import combinations +from typing import Any, Mapping + +import numpy as np +import pandas as pd + + +DEFAULT_MAX_SAMPLE_ROWS = 5_000 +DEFAULT_MAX_COLUMNS = 100 +DEFAULT_MAX_MISSINGNESS_COLUMNS = 25 + + +def _bounded_series(series: pd.Series, max_rows: int) -> tuple[pd.Series, bool]: + if len(series) <= max_rows: + return series, False + positions = np.linspace(0, len(series) - 1, num=max_rows, dtype=int) + positions = np.unique(positions) + return series.iloc[positions], True + + +def _bounded_frame(frame: pd.DataFrame, max_rows: int) -> tuple[pd.DataFrame, bool]: + if len(frame) <= max_rows: + return frame, False + positions = np.linspace(0, len(frame) - 1, num=max_rows, dtype=int) + positions = np.unique(positions) + return frame.iloc[positions], True + + +def _text_like(series: pd.Series) -> bool: + return bool( + pd.api.types.is_object_dtype(series) + or pd.api.types.is_string_dtype(series.dtype) + or isinstance(series.dtype, pd.CategoricalDtype) + ) + + +def _column_info(column_roles: Mapping[str, Any], column: str) -> dict[str, Any]: + value = column_roles.get(column, {}) + return dict(value) if isinstance(value, Mapping) else {} + + +def _profile_mapping(profile: Mapping[str, Any], key: str) -> dict[str, Any]: + value = profile.get(key, {}) + return dict(value) if isinstance(value, Mapping) else {} + + +def _primary_key_candidates( + df: pd.DataFrame, + column_roles: Mapping[str, Any], +) -> list[dict[str, Any]]: + rows = len(df) + if rows < 2: + return [] + + candidates: list[dict[str, Any]] = [] + for column in df.columns: + info = _column_info(column_roles, column) + unique_count = int(info.get("unique_count") or 0) + non_missing = int(info.get("non_missing_count") or 0) + if unique_count != rows or non_missing != rows: + continue + + roles = set(info.get("roles", [])) + confidence = "high" if "id_like" in roles else "medium" + candidates.append({ + "column": column, + "confidence": confidence, + "reason": ( + "Column is complete, unique for every row, and identifier-like." + if confidence == "high" + else "Column is complete and unique for every row." + ), + }) + + candidates.sort(key=lambda item: (item["confidence"] != "high", item["column"])) + return candidates + + +def _identifier_duplicates( + df: pd.DataFrame, + column_roles: Mapping[str, Any], +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for column in df.columns: + info = _column_info(column_roles, column) + if "id_like" not in set(info.get("roles", [])): + continue + clean = df[column].dropna() + if clean.empty: + continue + duplicate_mask = clean.duplicated(keep=False) + duplicate_rows = int(duplicate_mask.sum()) + if not duplicate_rows: + continue + duplicated_values = int(clean.loc[duplicate_mask].nunique(dropna=True)) + results.append({ + "column": column, + "duplicate_rows": duplicate_rows, + "duplicated_identifier_values": duplicated_values, + "severity": "high", + }) + return results + + +def _quasi_constants( + df: pd.DataFrame, + column_roles: Mapping[str, Any], + *, + max_rows: int, + threshold: float = 0.95, +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for column in df.columns: + info = _column_info(column_roles, column) + if int(info.get("unique_count") or 0) <= 1: + continue + sample, sampled = _bounded_series(df[column], max_rows) + clean = sample.dropna() + if len(clean) < 10: + continue + counts = clean.value_counts(dropna=True) + if counts.empty: + continue + ratio = float(counts.iloc[0] / len(clean)) + if ratio < threshold: + continue + results.append({ + "column": column, + "top_value": str(counts.index[0]), + "top_value_ratio": round(ratio, 4), + "sampled": sampled, + "sample_rows": int(len(sample)), + "severity": "medium" if ratio >= 0.99 else "low", + }) + return results + + +def _series_fingerprint(series: pd.Series) -> str: + hashed = pd.util.hash_pandas_object(series, index=False).to_numpy(dtype="uint64") + digest = hashlib.blake2b(digest_size=16) + digest.update(str(series.dtype).encode("utf-8")) + digest.update(hashed.tobytes()) + return digest.hexdigest() + + +def _duplicate_columns( + df: pd.DataFrame, + *, + max_rows: int, + max_columns: int, +) -> list[dict[str, Any]]: + columns = list(df.columns[:max_columns]) + sample, sampled = _bounded_frame(df[columns], max_rows) + buckets: dict[str, list[str]] = {} + + for column in columns: + fingerprint = _series_fingerprint(sample[column]) + buckets.setdefault(fingerprint, []).append(column) + + results: list[dict[str, Any]] = [] + for candidates in buckets.values(): + if len(candidates) < 2: + continue + anchor = candidates[0] + duplicates = [ + candidate + for candidate in candidates[1:] + if df[anchor].equals(df[candidate]) + ] + if duplicates: + results.append({ + "canonical_column": anchor, + "duplicate_columns": duplicates, + "sampled_for_fingerprint": sampled, + "confirmed_with_full_equality": True, + "severity": "medium", + }) + return results + + +def _normalise_numeric_text(values: pd.Series) -> pd.Series: + cleaned = values.str.replace(",", "", regex=False).str.strip() + cleaned = cleaned.str.replace(r"^[\$€£₹]\s*", "", regex=True) + cleaned = cleaned.str.replace(r"%$", "", regex=True) + return cleaned + + +def _coercion_candidates( + df: pd.DataFrame, + column_roles: Mapping[str, Any], + *, + max_rows: int, + threshold: float = 0.95, +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + + for column in df.columns: + series = df[column] + if not _text_like(series): + continue + + info = _column_info(column_roles, column) + roles = set(info.get("roles", [])) + sample, sampled = _bounded_series(series, max_rows) + clean = sample.dropna().astype("string").str.strip() + clean = clean[clean.ne("")] + if len(clean) < 10: + continue + + numeric_values = _normalise_numeric_text(clean) + numeric_ratio = float(pd.to_numeric(numeric_values, errors="coerce").notna().mean()) + + if numeric_ratio >= threshold and "id_like" not in roles: + semantic = info.get("semantic_type") + transform = "parse numeric strings" + if semantic == "currency": + transform = "remove currency symbols/separators, then parse numeric" + elif semantic == "percentage": + transform = "remove percent suffix, then parse numeric" + results.append({ + "column": column, + "suggested_type": "numeric", + "parse_ratio": round(numeric_ratio, 4), + "transform": transform, + "sampled": sampled, + "sample_rows": int(len(sample)), + "severity": "low", + }) + continue + + if numeric_ratio >= 0.50: + # Avoid treating mostly-numeric identifiers as dates. + continue + + parsed_dates = pd.to_datetime(clean, errors="coerce", format="mixed") + date_ratio = float(parsed_dates.notna().mean()) + if date_ratio >= threshold: + results.append({ + "column": column, + "suggested_type": "datetime", + "parse_ratio": round(date_ratio, 4), + "transform": "parse datetime strings", + "sampled": sampled, + "sample_rows": int(len(sample)), + "severity": "low", + }) + + return results + + +def _category_normalisation_issues( + df: pd.DataFrame, + column_roles: Mapping[str, Any], + *, + max_rows: int, + max_unique: int = 200, +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + + for column in df.columns: + series = df[column] + if not _text_like(series): + continue + info = _column_info(column_roles, column) + if "long_text" in set(info.get("roles", [])): + continue + + sample, sampled = _bounded_series(series, max_rows) + clean = sample.dropna().astype("string") + unique_values = clean.unique().tolist() + if len(unique_values) < 2 or len(unique_values) > max_unique: + continue + + groups: dict[str, list[str]] = {} + for raw in unique_values: + raw_text = str(raw) + canonical = raw_text.strip().casefold() + groups.setdefault(canonical, []).append(raw_text) + + variants = [ + {"canonical": canonical, "variants": sorted(set(raw_values))} + for canonical, raw_values in groups.items() + if len(set(raw_values)) > 1 + ] + if not variants: + continue + + variants.sort(key=lambda item: (-len(item["variants"]), item["canonical"])) + results.append({ + "column": column, + "variant_group_count": len(variants), + "groups": variants[:10], + "sampled": sampled, + "sample_rows": int(len(sample)), + "severity": "medium", + }) + + return results + + +def _blank_string_issues( + df: pd.DataFrame, + *, + max_rows: int, +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for column in df.columns: + series = df[column] + if not _text_like(series): + continue + sample, sampled = _bounded_series(series, max_rows) + non_null = sample.dropna().astype("string") + if non_null.empty: + continue + blank_count = int(non_null.str.strip().eq("").sum()) + if not blank_count: + continue + results.append({ + "column": column, + "blank_count_in_sample": blank_count, + "blank_ratio_in_sample": round(float(blank_count / len(non_null)), 4), + "sampled": sampled, + "sample_rows": int(len(sample)), + "severity": "medium", + }) + return results + + +def _infinity_issues( + df: pd.DataFrame, + *, + max_rows: int, +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + numeric_columns = df.select_dtypes(include=[np.number]).columns + for column in numeric_columns: + sample, sampled = _bounded_series(df[column], max_rows) + values = pd.to_numeric(sample, errors="coerce").to_numpy(dtype="float64") + count = int(np.isinf(values).sum()) + if not count: + continue + results.append({ + "column": column, + "infinite_count_in_sample": count, + "sampled": sampled, + "sample_rows": int(len(sample)), + "severity": "high", + }) + return results + + +def _mixed_object_types( + df: pd.DataFrame, + *, + max_rows: int, +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for column in df.columns: + series = df[column] + if not pd.api.types.is_object_dtype(series): + continue + sample, sampled = _bounded_series(series, max_rows) + clean = sample.dropna() + if len(clean) < 2: + continue + counts = clean.map(lambda value: type(value).__name__).value_counts() + if len(counts) <= 1: + continue + results.append({ + "column": column, + "python_types": {str(key): int(value) for key, value in counts.items()}, + "sampled": sampled, + "sample_rows": int(len(sample)), + "severity": "medium", + }) + return results + + +def _missingness_relationships( + df: pd.DataFrame, + profile: Mapping[str, Any], + *, + max_rows: int, + max_columns: int, + jaccard_threshold: float = 0.70, +) -> list[dict[str, Any]]: + missing_counts = _profile_mapping(profile, "missing_counts") + candidates = [ + column + for column in df.columns + if int(missing_counts.get(column) or 0) > 0 + ][:max_columns] + + if len(candidates) < 2: + return [] + + sample, sampled = _bounded_frame(df[candidates], max_rows) + masks = {column: sample[column].isna().to_numpy() for column in candidates} + relationships: list[dict[str, Any]] = [] + + for left, right in combinations(candidates, 2): + left_mask = masks[left] + right_mask = masks[right] + union = int(np.logical_or(left_mask, right_mask).sum()) + if not union: + continue + intersection = int(np.logical_and(left_mask, right_mask).sum()) + if intersection < 3: + continue + jaccard = intersection / union + if jaccard < jaccard_threshold: + continue + relationships.append({ + "columns": [left, right], + "co_missing_rows_in_sample": intersection, + "jaccard": round(float(jaccard), 4), + "sampled": sampled, + "sample_rows": int(len(sample)), + "severity": "medium" if jaccard >= 0.90 else "low", + }) + + relationships.sort(key=lambda item: (-item["jaccard"], item["columns"])) + return relationships[:30] + + +def run_quality_diagnostics( + df: pd.DataFrame, + *, + profile: Mapping[str, Any] | None = None, + column_roles: Mapping[str, Any] | None = None, + max_sample_rows: int = DEFAULT_MAX_SAMPLE_ROWS, + max_columns: int = DEFAULT_MAX_COLUMNS, + max_missingness_columns: int = DEFAULT_MAX_MISSINGNESS_COLUMNS, +) -> dict[str, Any]: + """Return a bounded, deterministic set of practical data-quality checks.""" + if max_sample_rows < 10: + raise ValueError("max_sample_rows must be at least 10.") + if max_columns < 1: + raise ValueError("max_columns must be at least 1.") + if max_missingness_columns < 2: + raise ValueError("max_missingness_columns must be at least 2.") + + if profile is None: + from framevitals.profiler import build_profile + + profile = build_profile(df) + if column_roles is None: + from framevitals.column_roles import infer_column_roles + + column_roles = infer_column_roles(df) + + selected_columns = list(df.columns[:max_columns]) + work = df[selected_columns] + selected_roles = { + column: _column_info(column_roles, column) + for column in selected_columns + } + + primary_keys = _primary_key_candidates(work, selected_roles) + identifier_duplicates = _identifier_duplicates(work, selected_roles) + quasi_constants = _quasi_constants( + work, + selected_roles, + max_rows=max_sample_rows, + ) + duplicate_columns = _duplicate_columns( + work, + max_rows=max_sample_rows, + max_columns=max_columns, + ) + coercions = _coercion_candidates( + work, + selected_roles, + max_rows=max_sample_rows, + ) + category_normalisation = _category_normalisation_issues( + work, + selected_roles, + max_rows=max_sample_rows, + ) + blank_strings = _blank_string_issues(work, max_rows=max_sample_rows) + infinities = _infinity_issues(work, max_rows=max_sample_rows) + mixed_types = _mixed_object_types(work, max_rows=max_sample_rows) + missingness_relationships = _missingness_relationships( + work, + profile, + max_rows=max_sample_rows, + max_columns=max_missingness_columns, + ) + + duplicate_rows = int(profile.get("duplicate_rows", 0) or 0) + checks = { + "primary_key_candidates": primary_keys, + "identifier_duplicates": identifier_duplicates, + "quasi_constant_columns": quasi_constants, + "duplicate_columns": duplicate_columns, + "coercion_candidates": coercions, + "category_normalisation": category_normalisation, + "blank_strings": blank_strings, + "infinite_values": infinities, + "mixed_object_types": mixed_types, + "missingness_relationships": missingness_relationships, + } + + issue_count = sum( + len(value) + for key, value in checks.items() + if key != "primary_key_candidates" + ) + + return { + "available": True, + "rows": int(len(df)), + "columns": int(len(df.columns)), + "columns_checked": len(selected_columns), + "truncated_columns": len(df.columns) > len(selected_columns), + "max_sample_rows": int(max_sample_rows), + "duplicate_rows": duplicate_rows, + "summary": { + "issue_groups": sum(bool(value) for key, value in checks.items() if key != "primary_key_candidates"), + "issue_count": issue_count + (1 if duplicate_rows else 0), + "primary_key_candidate_count": len(primary_keys), + }, + **checks, + } diff --git a/src/framevitals/quality_results.py b/src/framevitals/quality_results.py new file mode 100644 index 0000000..10a2303 --- /dev/null +++ b/src/framevitals/quality_results.py @@ -0,0 +1,235 @@ +"""Dict-compatible public results for checks, validation, drift, and quality gates.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + + +class _QualityResult(dict): + """Small mapping-compatible base with export conveniences.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + def to_dict(self) -> dict[str, Any]: + return deepcopy(dict(self)) + + def to_json( + self, + destination: str | Path | None = None, + *, + indent: int = 2, + ) -> str | Path: + rendered = json.dumps(self.to_dict(), indent=indent, default=str) + if destination is None: + return rendered + path = Path(destination) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered + "\n", encoding="utf-8") + return path + + +class CheckResult(_QualityResult): + """Dict-compatible result returned by :func:`framevitals.run_checks`.""" + + @property + def status(self) -> str: + return str(self.get("status", "unknown")) + + @property + def passed(self) -> bool: + return bool(self.get("passed", False)) + + @property + def findings(self) -> list[dict[str, Any]]: + value = self.get("findings", []) + return value if isinstance(value, list) else [] + + @property + def results(self) -> list[dict[str, Any]]: + value = self.get("results", []) + return value if isinstance(value, list) else [] + + def summary_text(self) -> str: + summary = self.get("summary", {}) + if not isinstance(summary, dict): + summary = {} + lines = [ + "FrameVitals custom checks", + f"Status {self.status.upper()}", + f"Checks {summary.get('checks', 0)}", + f"Passed {summary.get('passed', 0)}", + f"Errors {summary.get('errors', 0)}", + f"Warnings {summary.get('warnings', 0)}", + ] + if self.findings: + lines.extend(["", "Findings"]) + for finding in self.findings[:10]: + lines.append( + f"- [{str(finding.get('severity', 'error')).upper()}] " + f"{finding.get('title')}: {finding.get('message')}" + ) + return "\n".join(lines) + + +class ValidationResult(_QualityResult): + """Backward-compatible mapping returned by :func:`framevitals.validate`.""" + + @property + def valid(self) -> bool: + return bool(self.get("valid", False)) + + @property + def status(self) -> str: + return str(self.get("status", "unknown")) + + @property + def findings(self) -> list[dict[str, Any]]: + value = self.get("findings", []) + return value if isinstance(value, list) else [] + + def summary_text(self) -> str: + summary = self.get("summary", {}) + if not isinstance(summary, dict): + summary = {} + lines = [ + "FrameVitals validation", + f"Status {self.status.upper()}", + f"Columns checked {summary.get('columns_checked', 0)}", + f"Errors {summary.get('errors', 0)}", + f"Warnings {summary.get('warnings', 0)}", + ] + if self.findings: + lines.extend(["", "Top findings"]) + for finding in self.findings[:8]: + lines.append( + f"- [{str(finding.get('severity', 'error')).upper()}] " + f"{finding.get('column')}: {finding.get('message')}" + ) + return "\n".join(lines) + + +class DriftResult(_QualityResult): + """Backward-compatible mapping returned by :func:`framevitals.compare`.""" + + @property + def severity(self) -> str: + gate = self.get("gate", {}) + if isinstance(gate, dict): + return str(gate.get("severity", "unknown")) + return "unknown" + + @property + def status(self) -> str: + gate = self.get("gate", {}) + if isinstance(gate, dict): + return str(gate.get("status", "unknown")) + return "unknown" + + @property + def columns(self) -> list[dict[str, Any]]: + value = self.get("columns", []) + return value if isinstance(value, list) else [] + + def summary_text(self) -> str: + if not self.get("available"): + return ( + "FrameVitals drift\n" + "Status UNAVAILABLE\n" + f"Reason {self.get('reason')}" + ) + + summary = self.get("summary", {}) + schema = self.get("schema", {}) + if not isinstance(summary, dict): + summary = {} + if not isinstance(schema, dict): + schema = {} + lines = [ + "FrameVitals drift", + f"Gate {self.status.upper()}", + f"Severity {self.severity.upper()}", + f"Columns checked {summary.get('n_columns_compared', 0)}", + f"Added columns {len(schema.get('added_columns', []))}", + f"Removed columns {len(schema.get('removed_columns', []))}", + f"Type changes {len(schema.get('dtype_changes', []))}", + ] + notable = [ + entry + for entry in self.columns + if entry.get("drift_severity") in {"minor", "moderate", "severe"} + ] + if notable: + lines.extend(["", "Top drift"]) + for entry in notable[:8]: + lines.append( + f"- [{str(entry.get('drift_severity')).upper()}] {entry.get('column')}" + ) + return "\n".join(lines) + + +class GateResult(_QualityResult): + """Combined contract, custom-check, and drift quality-gate result.""" + + @property + def status(self) -> str: + return str(self.get("status", "unknown")) + + @property + def passed(self) -> bool: + return bool(self.get("passed", False)) + + @property + def reasons(self) -> list[str]: + value = self.get("reasons", []) + return value if isinstance(value, list) else [] + + @property + def checks_run(self) -> list[str]: + value = self.get("checks_run", []) + return value if isinstance(value, list) else [] + + def summary_text(self) -> str: + checks = self.get("checks", {}) + if not isinstance(checks, dict): + checks = {} + validation = checks.get("validation") + drift = checks.get("drift") + custom = checks.get("custom") + + lines = [ + "FrameVitals quality gate", + f"Status {self.status.upper()}", + f"Passed {'YES' if self.passed else 'NO'}", + ] + if isinstance(validation, dict): + lines.append( + f"Validation {str(validation.get('status', 'unknown')).upper()}" + ) + if isinstance(drift, dict): + drift_gate = drift.get("gate", {}) + if isinstance(drift_gate, dict): + lines.append( + "Drift " + f"{str(drift_gate.get('severity', 'unknown')).upper()}" + ) + if isinstance(custom, dict): + summary = custom.get("summary", {}) + if not isinstance(summary, dict): + summary = {} + lines.append( + "Custom checks " + f"{str(custom.get('status', 'unknown')).upper()} " + f"({summary.get('checks', 0)} run)" + ) + if self.reasons: + lines.extend(["", "Reasons"]) + for reason in self.reasons[:10]: + lines.append(f"- {reason}") + return "\n".join(lines) diff --git a/src/framevitals/relationship_graph.py b/src/framevitals/relationship_graph.py new file mode 100644 index 0000000..40dd000 --- /dev/null +++ b/src/framevitals/relationship_graph.py @@ -0,0 +1,367 @@ +"""Sparse feature-relationship discovery for wide numeric datasets. + +The engine deliberately avoids allocating a dense ``columns x columns`` matrix. +It builds compact SimHash-style signatures from a bounded row view, uses +locality-sensitive hash bands to generate a bounded candidate set, and verifies +only those candidates with Pearson correlation on the sampled observations. + +This Python implementation defines the semantics for the future Rust engine. +The native implementation can replace the scanning/signature/candidate kernels +without changing the result contract. +""" + +from __future__ import annotations + +from collections import Counter, OrderedDict +from dataclasses import dataclass +import heapq +import math +from typing import Any + +import numpy as np +import pandas as pd + + +@dataclass(slots=True) +class _UnionFind: + parent: dict[int, int] + size: dict[int, int] + + @classmethod + def create(cls) -> "_UnionFind": + return cls(parent={}, size={}) + + def _ensure(self, item: int) -> None: + if item not in self.parent: + self.parent[item] = item + self.size[item] = 1 + + def find(self, item: int) -> int: + self._ensure(item) + root = item + while self.parent[root] != root: + root = self.parent[root] + while self.parent[item] != item: + parent = self.parent[item] + self.parent[item] = root + item = parent + return root + + def union(self, left: int, right: int) -> None: + left_root = self.find(left) + right_root = self.find(right) + if left_root == right_root: + return + if self.size[left_root] < self.size[right_root]: + left_root, right_root = right_root, left_root + self.parent[right_root] = left_root + self.size[left_root] += self.size[right_root] + + +def _sample_positions(rows: int, max_rows: int) -> np.ndarray: + if rows <= max_rows: + return np.arange(rows, dtype=np.int64) + positions = np.linspace(0, rows - 1, num=max_rows, dtype=np.int64) + return np.unique(positions) + + +def _numeric_sample( + dataframe: pd.DataFrame, + column: Any, + positions: np.ndarray, +) -> np.ndarray: + series = pd.to_numeric(dataframe[column].iloc[positions], errors="coerce") + return series.to_numpy(dtype="float64", na_value=np.nan) + + +def _standardized_vector(values: np.ndarray, *, min_observations: int) -> np.ndarray | None: + finite = np.isfinite(values) + if int(finite.sum()) < min_observations: + return None + + clean = values[finite] + median = float(np.median(clean)) + work = np.where(finite, values, median).astype(np.float32, copy=False) + mean = float(work.mean()) + std = float(work.std()) + if not math.isfinite(std) or std <= 1e-12: + return None + return ((work - mean) / std).astype(np.float32, copy=False) + + +def _signature(vector: np.ndarray, hyperplanes: np.ndarray) -> int: + bits = np.asarray(vector @ hyperplanes >= 0, dtype=np.uint8) + packed = np.packbits(bits, bitorder="little").tobytes() + return int.from_bytes(packed, byteorder="little", signed=False) + + +def _absolute_band(signature: int, shift: int, band_bits: int) -> int: + """Canonicalize a signature band so positive/negative correlation can meet.""" + mask = (1 << band_bits) - 1 + value = (signature >> shift) & mask + complement = mask ^ value + return min(value, complement) + + +def _pearson_from_sample(left: np.ndarray, right: np.ndarray) -> tuple[float | None, int]: + mask = np.isfinite(left) & np.isfinite(right) + overlap = int(mask.sum()) + if overlap < 10: + return None, overlap + + x = left[mask].astype(np.float64, copy=False) + y = right[mask].astype(np.float64, copy=False) + x = x - x.mean() + y = y - y.mean() + denominator = float(np.linalg.norm(x) * np.linalg.norm(y)) + if denominator <= 1e-12: + return None, overlap + correlation = float(np.dot(x, y) / denominator) + if not math.isfinite(correlation): + return None, overlap + return max(-1.0, min(1.0, correlation)), overlap + + +class _SampleCache: + """Tiny LRU cache so candidate verification does not retain the wide sample.""" + + def __init__( + self, + dataframe: pd.DataFrame, + columns: list[Any], + positions: np.ndarray, + *, + max_columns: int, + ) -> None: + self.dataframe = dataframe + self.columns = columns + self.positions = positions + self.max_columns = max_columns + self.cache: OrderedDict[int, np.ndarray] = OrderedDict() + + def get(self, index: int) -> np.ndarray: + if index in self.cache: + value = self.cache.pop(index) + self.cache[index] = value + return value + + value = _numeric_sample( + self.dataframe, + self.columns[index], + self.positions, + ) + self.cache[index] = value + if len(self.cache) > self.max_columns: + self.cache.popitem(last=False) + return value + + +def build_numeric_relationship_graph( + dataframe: pd.DataFrame, + *, + max_sample_rows: int = 512, + projections: int = 64, + band_bits: int = 16, + neighbors_per_bucket: int = 4, + max_bucket_members: int = 64, + max_candidate_pairs: int = 250_000, + min_abs_correlation: float = 0.80, + max_edges_returned: int = 5_000, + sample_cache_columns: int = 128, + random_state: int = 42, +) -> dict[str, Any]: + """Discover strong numeric relationships without a dense correlation matrix. + + Candidate generation is approximate; candidate verification is a normal + Pearson calculation on the bounded row view. The result always reports the + candidate/row budgets and whether either was truncated. + """ + if not isinstance(dataframe, pd.DataFrame): + raise TypeError("dataframe must be a pandas DataFrame.") + if max_sample_rows < 20: + raise ValueError("max_sample_rows must be at least 20.") + if projections < 16 or projections > 256 or projections % 8: + raise ValueError("projections must be a multiple of 8 between 16 and 256.") + if band_bits < 4 or projections % band_bits: + raise ValueError("band_bits must divide projections and be at least 4.") + if neighbors_per_bucket < 1: + raise ValueError("neighbors_per_bucket must be at least 1.") + if max_bucket_members < neighbors_per_bucket: + raise ValueError("max_bucket_members must be >= neighbors_per_bucket.") + if max_candidate_pairs < 1: + raise ValueError("max_candidate_pairs must be at least 1.") + if not 0 < min_abs_correlation <= 1: + raise ValueError("min_abs_correlation must be in (0, 1].") + if max_edges_returned < 1: + raise ValueError("max_edges_returned must be at least 1.") + if sample_cache_columns < 2: + raise ValueError("sample_cache_columns must be at least 2.") + + columns = dataframe.select_dtypes(include=[np.number]).columns.tolist() + node_count = len(columns) + if node_count < 2: + return { + "available": False, + "reason": "Need at least two numeric columns.", + "nodes": node_count, + "edges": [], + } + + positions = _sample_positions(len(dataframe), max_sample_rows) + sample_rows = int(len(positions)) + min_observations = min(20, max(10, sample_rows // 4)) + + rng = np.random.default_rng(random_state) + hyperplanes = rng.choice( + np.array([-1.0, 1.0], dtype=np.float32), + size=(sample_rows, projections), + ) + hyperplanes /= math.sqrt(max(sample_rows, 1)) + + signatures: dict[int, int] = {} + skipped_columns: list[str] = [] + for index, column in enumerate(columns): + raw = _numeric_sample(dataframe, column, positions) + vector = _standardized_vector(raw, min_observations=min_observations) + if vector is None: + skipped_columns.append(str(column)) + continue + signatures[index] = _signature(vector, hyperplanes) + + band_count = projections // band_bits + buckets: dict[tuple[int, int], list[int]] = {} + candidates: set[tuple[int, int]] = set() + candidate_truncated = False + + for index in sorted(signatures): + signature = signatures[index] + for band in range(band_count): + key = ( + band, + _absolute_band(signature, band * band_bits, band_bits), + ) + members = buckets.setdefault(key, []) + for other in members[-neighbors_per_bucket:]: + pair = (other, index) if other < index else (index, other) + candidates.add(pair) + if len(candidates) >= max_candidate_pairs: + candidate_truncated = True + break + if len(members) >= max_bucket_members: + del members[: len(members) - max_bucket_members + 1] + members.append(index) + if candidate_truncated: + break + if candidate_truncated: + break + + total_possible_pairs = node_count * (node_count - 1) // 2 + cache = _SampleCache( + dataframe, + columns, + positions, + max_columns=sample_cache_columns, + ) + + union_find = _UnionFind.create() + degrees: Counter[int] = Counter() + verified_edges = 0 + top_edge_heap: list[tuple[float, int, int, float, int]] = [] + + for left, right in sorted(candidates): + correlation, overlap = _pearson_from_sample(cache.get(left), cache.get(right)) + if correlation is None or abs(correlation) < min_abs_correlation: + continue + + verified_edges += 1 + union_find.union(left, right) + degrees[left] += 1 + degrees[right] += 1 + item = (abs(correlation), left, right, correlation, overlap) + if len(top_edge_heap) < max_edges_returned: + heapq.heappush(top_edge_heap, item) + elif item[0] > top_edge_heap[0][0]: + heapq.heapreplace(top_edge_heap, item) + + edges = [ + { + "source": str(columns[left]), + "target": str(columns[right]), + "correlation": round(float(correlation), 6), + "abs_correlation": round(float(score), 6), + "overlap": int(overlap), + } + for score, left, right, correlation, overlap in sorted( + top_edge_heap, + key=lambda item: (-item[0], str(columns[item[1]]), str(columns[item[2]])), + ) + ] + + component_members: dict[int, list[int]] = {} + for index in union_find.parent: + root = union_find.find(index) + component_members.setdefault(root, []).append(index) + + components = sorted( + component_members.values(), + key=lambda members: (-len(members), [str(columns[index]) for index in members]), + ) + component_summary = [ + { + "size": len(members), + "members": [str(columns[index]) for index in sorted(members)[:20]], + "members_truncated": len(members) > 20, + } + for members in components[:50] + if len(members) >= 2 + ] + + connected_nodes = len({index for pair in candidates for index in pair if degrees[index]}) + high_degree = sorted( + ( + {"column": str(columns[index]), "degree": int(degree)} + for index, degree in degrees.items() + ), + key=lambda item: (-item["degree"], item["column"]), + )[:25] + + return { + "available": True, + "method": "bounded_simhash_lsh_then_pearson", + "nodes": node_count, + "usable_signature_nodes": len(signatures), + "skipped_nodes": len(skipped_columns), + "skipped_columns": skipped_columns[:25], + "sample": { + "source_rows": int(len(dataframe)), + "sample_rows": sample_rows, + "sampled": len(dataframe) > sample_rows, + "strategy": "deterministic_evenly_spaced", + }, + "candidate_generation": { + "projections": projections, + "band_bits": band_bits, + "bands": band_count, + "neighbors_per_bucket": neighbors_per_bucket, + "candidate_pairs": len(candidates), + "max_candidate_pairs": max_candidate_pairs, + "truncated": candidate_truncated, + "dense_pairs_avoided": max(total_possible_pairs - len(candidates), 0), + "total_possible_dense_pairs": total_possible_pairs, + }, + "verification": { + "method": "sample_pearson", + "min_abs_correlation": float(min_abs_correlation), + "verified_relationships": verified_edges, + "edges_returned": len(edges), + "edges_truncated": verified_edges > len(edges), + }, + "graph": { + "connected_nodes": connected_nodes, + "isolated_or_unverified_nodes": max(node_count - connected_nodes, 0), + "component_count": len(component_summary), + "components": component_summary, + "high_degree_nodes": high_degree, + }, + "edges": edges, + } diff --git a/src/framevitals/reporting/__init__.py b/src/framevitals/reporting/__init__.py new file mode 100644 index 0000000..102ce01 --- /dev/null +++ b/src/framevitals/reporting/__init__.py @@ -0,0 +1,14 @@ +"""Human-facing renderers for FrameVitals results. + +Renderers are intentionally dependency-light and consume the stable result +shape rather than re-running analysis. +""" + +from framevitals.reporting.html import render_html_report, render_notebook_summary +from framevitals.reporting.terminal import render_terminal_summary + +__all__ = [ + "render_html_report", + "render_notebook_summary", + "render_terminal_summary", +] diff --git a/src/framevitals/reporting/html.py b/src/framevitals/reporting/html.py new file mode 100644 index 0000000..85d1644 --- /dev/null +++ b/src/framevitals/reporting/html.py @@ -0,0 +1,217 @@ +"""Self-contained HTML rendering for FrameVitals analysis results.""" + +from __future__ import annotations + +import json +from html import escape +from typing import Any, Mapping + + +def _text(value: Any) -> str: + return escape(str(value if value is not None else "")) + + +def _score(value: Any) -> float: + try: + return max(0.0, min(100.0, float(value))) + except (TypeError, ValueError): + return 0.0 + + +def _severity_class(value: Any) -> str: + severity = str(value or "info").lower() + if severity in {"critical", "high", "medium", "low", "info"}: + return severity + return "info" + + +def _column_rows(result: Mapping[str, Any]) -> str: + profile = result.get("profile", {}) or {} + roles = result.get("column_roles", {}) or {} + columns = profile.get("columns", []) or [] + dtypes = profile.get("dtypes", {}) or {} + missing = profile.get("missing_percent", {}) or {} + + rows: list[str] = [] + for name in columns: + role_info = roles.get(name, {}) or {} + role_text = ", ".join(role_info.get("roles", [])[:6]) + unique_count = role_info.get("unique_count", "") + rows.append( + "" + f"{_text(name)}" + f"{_text(dtypes.get(name, ''))}" + f"{_text(missing.get(name, ''))}%" + f"{_text(unique_count)}" + f"{_text(role_text)}" + "" + ) + return "".join(rows) + + +def _finding_cards(result: Mapping[str, Any]) -> str: + findings = result.get("findings", []) or [] + if not findings: + return '
No actionable findings from the current signal layer.
' + + cards: list[str] = [] + for finding in findings: + severity = _severity_class(finding.get("severity")) + recommendation = finding.get("recommendation") or "" + recommendation_html = ( + f'
Next action
{_text(recommendation)}
' + if recommendation + else "" + ) + cards.append( + f'
' + f'
{_text(severity.upper())}' + f'{_text(finding.get("code", ""))}
' + f'

{_text(finding.get("title", "Finding"))}

' + f'

{_text(finding.get("evidence", ""))}

' + f'{recommendation_html}' + '
' + ) + return "".join(cards) + + +def render_notebook_summary(result: Mapping[str, Any]) -> str: + """Return a compact notebook-safe HTML representation.""" + profile = result.get("profile", {}) or {} + shape = profile.get("shape", {}) or {} + health = result.get("health", {}) or {} + ml = result.get("ml_readiness", {}) or {} + findings = result.get("findings", []) or [] + return f""" +
+
+
FrameVitals
{_text(result.get('filename', ''))}
+
{_text(shape.get('rows','?'))} rows × {_text(shape.get('columns','?'))} columns
+
+
+
Health
{_text(health.get('overall_score','n/a'))}/100
{_text(health.get('label',''))}
+
ML readiness
{_text(ml.get('score','n/a'))}/100
{_text(ml.get('label',''))}
+
Findings
{len(findings)}
actionable
+
+
+""".strip() + + +def render_html_report(result: Mapping[str, Any]) -> str: + """Render a complete, self-contained HTML analysis report.""" + profile = result.get("profile", {}) or {} + shape = profile.get("shape", {}) or {} + health = result.get("health", {}) or {} + ml = result.get("ml_readiness", {}) or {} + findings = result.get("findings", []) or [] + recommendations = [] + seen: set[str] = set() + for finding in findings: + recommendation = str(finding.get("recommendation") or "").strip() + if recommendation and recommendation not in seen: + seen.add(recommendation) + recommendations.append(recommendation) + + health_score = _score(health.get("overall_score")) + ml_score = _score(ml.get("score")) + missing_percent = health.get("details", {}).get("missing_percent", 0) + duplicate_percent = profile.get("duplicate_percent", 0) + memory = profile.get("memory_usage_mb", "n/a") + total_ms = (result.get("timings_ms", {}) or {}).get("total") + duration = f"{float(total_ms) / 1000:.2f}s" if isinstance(total_ms, (int, float)) else "n/a" + + recommendation_items = "".join( + f"
  • {_text(item)}
    • " for item in recommendations + ) or "
  • No additional remediation steps are required by the current finding layer.
    • " + + raw_json = escape(json.dumps(dict(result), indent=2, default=str)) + + return f""" + + + + +FrameVitals Report — {_text(result.get('filename', 'dataset'))} + + + +
    +
    +
    FrameVitals analysis report
    +

    {_text(result.get('filename', ''))}

    +

    Data health, structure, ML readiness, and actionable diagnostics in one report.

    +
    + {_text(shape.get('rows','?'))} rows + {_text(shape.get('columns','?'))} columns + {_text(result.get('analysis_mode','unknown'))} mode + {len(findings)} findings + {_text(duration)} runtime +
    +
    + +
    +
    Data health
    {_text(health.get('overall_score','n/a'))} / 100
    {_text(health.get('label',''))}
    +
    ML readiness
    {_text(ml.get('score','n/a'))} / 100
    {_text(ml.get('label',''))}
    +
    Actionable findings
    {len(findings)}
    Normalized from FrameVitals' deterministic signal layer.
    + +
    Missing cells
    {_text(missing_percent)}%
    +
    Duplicate rows
    {_text(duplicate_percent)}%
    +
    Memory
    {_text(memory)} MB
    +
    Runtime
    {_text(duration)}
    + +
    Findings

    What needs attention

    {_finding_cards(result)}
    + +
    Recommendations

    Suggested next actions

      {recommendation_items}
    + +
    Columns

    Dataset structure

    {_column_rows(result)}
    ColumndtypeMissingUniqueRoles
    + +
    Complete result
    Inspect raw JSON
    {raw_json}
    +
    +
    Generated locally by FrameVitals. This HTML file is self-contained and does not require a server.
    +
    + + +""" diff --git a/src/framevitals/reporting/terminal.py b/src/framevitals/reporting/terminal.py new file mode 100644 index 0000000..b8f61ad --- /dev/null +++ b/src/framevitals/reporting/terminal.py @@ -0,0 +1,80 @@ +"""Dependency-light terminal rendering for FrameVitals results.""" + +from __future__ import annotations + +from typing import Any, Mapping + + +def _score_bar(value: Any, width: int = 24) -> str: + try: + score = max(0.0, min(100.0, float(value))) + except (TypeError, ValueError): + return "[" + "?" * width + "]" + filled = round(score / 100 * width) + return "[" + "#" * filled + "-" * (width - filled) + "]" + + +def _fmt_score(value: Any) -> str: + try: + return f"{float(value):.1f}/100" + except (TypeError, ValueError): + return "n/a" + + +def _clean_line(value: Any, *, max_length: int = 110) -> str: + text = " ".join(str(value or "").split()) + if len(text) <= max_length: + return text + return text[: max_length - 1].rstrip() + "…" + + +def render_terminal_summary(result: Mapping[str, Any]) -> str: + """Render a compact report suitable for interactive terminal output.""" + profile = result.get("profile", {}) or {} + shape = profile.get("shape", {}) or {} + health = result.get("health", {}) or {} + ml = result.get("ml_readiness", {}) or {} + findings = result.get("findings", []) or [] + timings = result.get("timings_ms", {}) or {} + + rows = shape.get("rows", "?") + columns = shape.get("columns", "?") + health_score = health.get("overall_score") + ml_score = ml.get("score") + + lines = [ + "FrameVitals Analysis", + "=" * 72, + f"Dataset {result.get('filename', '')}", + f"Mode {result.get('analysis_mode', 'unknown')}", + f"Shape {rows} rows x {columns} columns", + f"Memory {profile.get('memory_usage_mb', 'n/a')} MB", + "", + f"Health {_score_bar(health_score)} {_fmt_score(health_score)} {health.get('label', '')}", + f"ML readiness {_score_bar(ml_score)} {_fmt_score(ml_score)} {ml.get('label', '')}", + "", + f"Findings {len(findings)} actionable issue(s)", + ] + + if findings: + for finding in findings[:6]: + severity = str(finding.get("severity", "info")).upper() + title = _clean_line(finding.get("title", "Finding"), max_length=42) + evidence = _clean_line(finding.get("evidence", ""), max_length=88) + lines.append(f" [{severity:<8}] {title}") + if evidence: + lines.append(f" {evidence}") + if len(findings) > 6: + lines.append(f" ... and {len(findings) - 6} more finding(s)") + else: + lines.append(" No actionable findings from the current signal layer.") + + total_ms = timings.get("total") + if isinstance(total_ms, (int, float)): + lines.extend(["", f"Completed in {total_ms / 1000:.2f}s"]) + + lines.extend([ + "=" * 72, + "Use result.to_html(...) or CLI --output to keep the complete report.", + ]) + return "\n".join(lines) diff --git a/src/framevitals/result.py b/src/framevitals/result.py new file mode 100644 index 0000000..95e7b9b --- /dev/null +++ b/src/framevitals/result.py @@ -0,0 +1,345 @@ +"""Public result objects for FrameVitals. + +FrameVitals result objects intentionally subclass :class:`dict` during the 0.x +series. That preserves existing mapping behaviour and JSON compatibility while +adding discoverable helpers for notebooks, applications, reports, and CI. +""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + +from framevitals.findings import ( + findings_from_quality_diagnostics, + findings_from_signals, + findings_from_target_intelligence, + merge_findings, + recommendations_from_findings, +) + + +class ColumnResult(dict): + """Structured view of one analyzed column.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + +class DiagnosticResult(dict): + """Dict-compatible result returned by focused diagnostic APIs. + + The diagnostic name is stored as Python-side metadata rather than inserted + into the mapping, so wrapping an existing focused payload does not mutate its + JSON schema during the 0.x compatibility window. + """ + + def __init__( + self, + *args, + diagnostic: str = "diagnostic", + **kwargs, + ) -> None: + super().__init__(*args, **kwargs) + self._diagnostic = str(diagnostic or "diagnostic") + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + @property + def diagnostic(self) -> str: + return self._diagnostic + + @property + def dataset_name(self) -> str | None: + value = self.get("dataset_name") + return str(value) if value is not None else None + + @property + def execution(self) -> dict[str, Any]: + value = self.get("execution", {}) + return value if isinstance(value, dict) else {} + + @property + def source(self) -> dict[str, Any]: + value = self.get("source") + if not isinstance(value, dict): + value = self.execution.get("source") + if not isinstance(value, dict): + value = self.get("source_metadata") + return value if isinstance(value, dict) else {} + + @property + def available(self) -> bool: + value = self.get("available") + return True if value is None else bool(value) + + def to_dict(self) -> dict[str, Any]: + """Return a detached plain dictionary without Python-side metadata.""" + return deepcopy(dict(self)) + + def to_json( + self, + destination: str | Path | None = None, + *, + indent: int = 2, + ) -> str | Path: + """Serialize the complete diagnostic result or write it to a file.""" + rendered = json.dumps(self.to_dict(), indent=indent, default=str) + if destination is None: + return rendered + path = Path(destination) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered + "\n", encoding="utf-8") + return path + + def summary(self) -> dict[str, Any]: + """Return a compact operation-agnostic execution summary.""" + execution = self.execution + shape = self.get("shape") + if not isinstance(shape, dict): + shape = {} + source = self.source + return { + "diagnostic": self.diagnostic, + "dataset_name": self.dataset_name, + "available": self.available, + "shape": dict(shape), + "method": execution.get("method"), + "full_materialization": execution.get("full_materialization"), + "sampled": execution.get("sampled"), + "source_rows": execution.get("source_rows", source.get("rows")), + "source_columns": execution.get("source_columns", source.get("columns")), + "sample_rows": execution.get("sample_rows"), + "execution_schema_version": execution.get("execution_schema_version"), + } + + def summary_text(self) -> str: + """Render a compact terminal-friendly focused-diagnostic summary.""" + summary = self.summary() + lines = [ + f"FrameVitals {self.diagnostic}", + f"Dataset {summary.get('dataset_name') or 'unknown'}", + f"Available {'yes' if summary.get('available') else 'no'}", + ] + if summary.get("method") is not None: + lines.append(f"Method {summary['method']}") + if summary.get("source_rows") is not None: + lines.append(f"Source rows {summary['source_rows']}") + if summary.get("sample_rows") is not None: + lines.append(f"Sample rows {summary['sample_rows']}") + if summary.get("sampled") is not None: + lines.append( + f"Sampled {'yes' if summary.get('sampled') else 'no'}" + ) + if summary.get("full_materialization") is not None: + lines.append( + "Materialized " + f"{'yes' if summary.get('full_materialization') else 'no'}" + ) + return "\n".join(lines) + + +class AnalysisResult(dict): + """Backward-compatible result returned by :func:`framevitals.analyze`.""" + + schema_version = "1" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.setdefault("result_schema_version", self.schema_version) + if "findings" not in self: + signals = self.get("signals", []) + if not isinstance(signals, list): + signals = [] + self["findings"] = merge_findings( + findings_from_signals(signals), + findings_from_quality_diagnostics(self.get("quality_diagnostics")), + findings_from_target_intelligence(self.get("target_intelligence")), + ) + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + @property + def findings(self) -> list[dict[str, Any]]: + value = self.get("findings", []) + return value if isinstance(value, list) else [] + + @property + def recommendations(self) -> list[str]: + return recommendations_from_findings(self.findings) + + @property + def health(self) -> dict[str, Any]: + value = self.get("health", {}) + return value if isinstance(value, dict) else {} + + @property + def ml_readiness(self) -> dict[str, Any]: + value = self.get("ml_readiness", {}) + return value if isinstance(value, dict) else {} + + @property + def shape(self) -> dict[str, Any]: + profile = self.get("profile", {}) + if not isinstance(profile, dict): + return {} + shape = profile.get("shape", {}) + return shape if isinstance(shape, dict) else {} + + def to_dict(self) -> dict[str, Any]: + """Return a detached plain-dictionary copy of the complete result.""" + return deepcopy(dict(self)) + + def summary(self) -> dict[str, Any]: + """Return a concise, stable high-level summary of the analysis.""" + health = self.health + ml_readiness = self.ml_readiness + timings = self.get("timings_ms", {}) + if not isinstance(timings, dict): + timings = {} + + severity_counts: dict[str, int] = {} + for finding in self.findings: + severity = str(finding.get("severity") or "unknown") + severity_counts[severity] = severity_counts.get(severity, 0) + 1 + + return { + "dataset_id": self.get("dataset_id"), + "filename": self.get("filename"), + "analysis_mode": self.get("analysis_mode"), + "result_schema_version": self.get("result_schema_version"), + "shape": dict(self.shape), + "health": { + "overall_score": health.get("overall_score"), + "label": health.get("label"), + }, + "ml_readiness": { + "score": ml_readiness.get("score"), + "label": ml_readiness.get("label"), + }, + "finding_count": len(self.findings), + "finding_severity_counts": severity_counts, + "artifacts_enabled": bool(self.get("artifacts_enabled", False)), + "total_ms": timings.get("total"), + } + + def column(self, name: str) -> ColumnResult: + """Return the combined profile, role, semantic, and quality view for one column.""" + profile = self.get("profile", {}) + roles = self.get("column_roles", {}) + if not isinstance(profile, dict): + profile = {} + if not isinstance(roles, dict): + roles = {} + + columns = profile.get("columns", []) + if name not in columns and name not in roles: + raise KeyError(f"Column not found in analysis result: {name}") + + role_info = roles.get(name, {}) + if not isinstance(role_info, dict): + role_info = {} + + numeric_summary = profile.get("numeric_summary", {}) + categorical_summary = profile.get("categorical_summary", {}) + correlations = profile.get("correlations", {}) + if not isinstance(numeric_summary, dict): + numeric_summary = {} + if not isinstance(categorical_summary, dict): + categorical_summary = {} + if not isinstance(correlations, dict): + correlations = {} + + quality_findings = [ + finding + for finding in self.findings + if str(finding.get("code", "")).startswith("quality.") + and ( + f".{name.lower().replace(' ', '_')}" in str(finding.get("code", "")) + or name in str(finding.get("title", "")) + ) + ] + + payload = { + "name": name, + "dtype": profile.get("dtypes", {}).get(name), + "roles": list(role_info.get("roles", [])), + "semantic_type": role_info.get("semantic_type"), + "semantic_candidates": list(role_info.get("semantic_candidates", [])), + "semantic_sample_size": role_info.get("semantic_sample_size", 0), + "missing_count": profile.get("missing_counts", {}).get(name), + "missing_percent": profile.get("missing_percent", {}).get(name), + "unique_count": role_info.get("unique_count"), + "unique_ratio": role_info.get("unique_ratio"), + "non_missing_count": role_info.get("non_missing_count"), + "is_numeric": role_info.get("is_numeric"), + "is_categorical": role_info.get("is_categorical"), + "numeric_summary": numeric_summary.get(name), + "categorical_summary": categorical_summary.get(name), + "correlations": correlations.get(name, {}), + "quality_findings": quality_findings, + } + return ColumnResult(payload) + + def to_json( + self, + destination: str | Path | None = None, + *, + indent: int = 2, + ) -> str | Path: + """Serialize the complete result to JSON or write it to ``destination``.""" + rendered = json.dumps(self.to_dict(), indent=indent, default=str) + if destination is None: + return rendered + + path = Path(destination) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered + "\n", encoding="utf-8") + return path + + def summary_text(self) -> str: + """Render a compact human-readable terminal summary.""" + from framevitals.reporting.terminal import render_terminal_summary + + return render_terminal_summary(self) + + def to_html(self, destination: str | Path | None = None) -> str | Path: + """Render a self-contained HTML report and optionally write it to disk.""" + from framevitals.reporting.html import render_html_report + + rendered = render_html_report(self) + if destination is None: + return rendered + + path = Path(destination) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered, encoding="utf-8") + return path + + def snapshot(self, destination: str | Path | None = None): + """Create a compact versioned monitoring snapshot from this result.""" + from framevitals.snapshots import create_snapshot + + snapshot = create_snapshot(self) + if destination is not None: + snapshot.to_json(destination) + return snapshot + + def _repr_html_(self) -> str: + """Provide a compact rich representation in Jupyter-compatible clients.""" + from framevitals.reporting.html import render_notebook_summary + + return render_notebook_summary(self) diff --git a/src/framevitals/semantic_types.py b/src/framevitals/semantic_types.py new file mode 100644 index 0000000..c3582a5 --- /dev/null +++ b/src/framevitals/semantic_types.py @@ -0,0 +1,131 @@ +"""Deterministic semantic type inference for text-like columns. + +The detector intentionally uses bounded samples so semantic typing remains +cheap on large datasets. It augments, rather than replaces, dtype and name-based +column role inference. +""" + +from __future__ import annotations + +import ipaddress +import json +import re +import uuid +from typing import Callable + +import pandas as pd + + +_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") +_URL_RE = re.compile(r"^(?:https?://|www\.)[^\s]+$", re.IGNORECASE) +_PHONE_RE = re.compile(r"^\+?[0-9][0-9\s().-]{6,}$") +_PERCENT_RE = re.compile(r"^[+-]?(?:\d+(?:\.\d+)?|\.\d+)\s*%$") +_CURRENCY_RE = re.compile( + r"^(?:[$€£₹¥]\s*[+-]?[\d,.]+|(?:USD|EUR|GBP|INR|JPY)\s+[+-]?[\d,.]+)$", + re.IGNORECASE, +) +_BOOL_TOKEN_RE = re.compile( + r"^(?:true|false|yes|no|y|n|on|off)$", + re.IGNORECASE, +) + + +def _matches_uuid(value: str) -> bool: + try: + uuid.UUID(value) + return True + except (ValueError, AttributeError, TypeError): + return False + + +def _matches_ip(value: str) -> bool: + try: + ipaddress.ip_address(value) + return True + except ValueError: + return False + + +def _matches_phone(value: str) -> bool: + if not _PHONE_RE.match(value): + return False + digit_count = sum(character.isdigit() for character in value) + return 7 <= digit_count <= 15 + + +def _matches_json(value: str) -> bool: + stripped = value.strip() + if not stripped.startswith(("{", "[")): + return False + try: + parsed = json.loads(stripped) + except (json.JSONDecodeError, TypeError): + return False + return isinstance(parsed, (dict, list)) + + +def _ratio(sample: list[str], predicate: Callable[[str], bool]) -> float: + if not sample: + return 0.0 + return sum(1 for value in sample if predicate(value)) / len(sample) + + +def infer_semantic_types( + series: pd.Series, + *, + max_samples: int = 100, + threshold: float = 0.70, +) -> dict: + """Infer semantic value types from a bounded non-null sample. + + Returns a dictionary with ``primary``, ranked ``candidates``, and + ``sample_size``. Empty/unsupported columns return no candidates. + """ + if max_samples < 1: + raise ValueError("max_samples must be at least 1.") + if not 0 < threshold <= 1: + raise ValueError("threshold must be in the interval (0, 1].") + + is_text = ( + pd.api.types.is_object_dtype(series) + or pd.api.types.is_string_dtype(series.dtype) + or isinstance(series.dtype, pd.CategoricalDtype) + ) + if not is_text: + return {"primary": None, "candidates": [], "sample_size": 0} + + sample = [ + value.strip() + for value in series.dropna().astype(str).head(max_samples).tolist() + if value.strip() + ] + if not sample: + return {"primary": None, "candidates": [], "sample_size": 0} + + checks: list[tuple[str, Callable[[str], bool]]] = [ + ("email", lambda value: bool(_EMAIL_RE.match(value))), + ("url", lambda value: bool(_URL_RE.match(value))), + ("uuid", _matches_uuid), + ("ip_address", _matches_ip), + ("phone", _matches_phone), + ("percentage", lambda value: bool(_PERCENT_RE.match(value))), + ("currency", lambda value: bool(_CURRENCY_RE.match(value))), + ("json", _matches_json), + ("boolean_token", lambda value: bool(_BOOL_TOKEN_RE.match(value))), + ] + + candidates = [] + for semantic_type, predicate in checks: + confidence = _ratio(sample, predicate) + if confidence >= threshold: + candidates.append({ + "type": semantic_type, + "confidence": round(confidence, 4), + }) + + candidates.sort(key=lambda item: (-item["confidence"], item["type"])) + return { + "primary": candidates[0]["type"] if candidates else None, + "candidates": candidates, + "sample_size": len(sample), + } diff --git a/src/framevitals/snapshots.py b/src/framevitals/snapshots.py new file mode 100644 index 0000000..d22ee67 --- /dev/null +++ b/src/framevitals/snapshots.py @@ -0,0 +1,327 @@ +"""Compact versioned snapshots and lightweight local monitoring history.""" + +from __future__ import annotations + +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + + +SNAPSHOT_SCHEMA_VERSION = "1" + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _number(value: Any) -> float | None: + try: + if value is None: + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _state_payload(result: Mapping[str, Any]) -> dict[str, Any]: + profile = _as_mapping(result.get("profile")) + health = _as_mapping(result.get("health")) + ml = _as_mapping(result.get("ml_readiness")) + findings = result.get("findings", []) + if not isinstance(findings, list): + findings = [] + + return { + "result_schema_version": result.get("result_schema_version"), + "analysis_mode": result.get("analysis_mode"), + "dataset": { + "shape": dict(_as_mapping(profile.get("shape"))), + "dtypes": dict(_as_mapping(profile.get("dtypes"))), + "missing_percent": dict(_as_mapping(profile.get("missing_percent"))), + "duplicate_percent": profile.get("duplicate_percent"), + "memory_usage_mb": profile.get("memory_usage_mb"), + }, + "health": { + "overall_score": health.get("overall_score"), + "label": health.get("label"), + "components": dict(_as_mapping(health.get("components"))), + }, + "ml_readiness": { + "score": ml.get("score"), + "label": ml.get("label"), + "issues": dict(_as_mapping(ml.get("issues"))), + }, + "finding_codes": sorted( + str(item.get("code")) + for item in findings + if isinstance(item, Mapping) and item.get("code") + ), + "config": dict(_as_mapping(result.get("config"))), + } + + +def _fingerprint(state: Mapping[str, Any]) -> str: + canonical = json.dumps( + dict(state), + sort_keys=True, + separators=(",", ":"), + default=str, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _created_at(snapshot: Mapping[str, Any]) -> datetime: + value = snapshot.get("created_at") + if not isinstance(value, str): + raise ValueError("Snapshot is missing a valid created_at timestamp.") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError(f"Snapshot has an invalid created_at timestamp: {value!r}") from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _safe_label(value: str | None) -> str | None: + if value is None: + return None + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", value.strip()).strip("-._") + return cleaned[:80] or None + + +class AnalysisSnapshot(dict): + """Small JSON-friendly state record for monitoring and history.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: + raise AttributeError(name) from exc + + def to_json( + self, + destination: str | Path | None = None, + *, + indent: int = 2, + ) -> str | Path: + rendered = json.dumps(dict(self), indent=indent, default=str) + if destination is None: + return rendered + path = Path(destination) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(rendered + "\n", encoding="utf-8") + return path + + def diff(self, other: Mapping[str, Any]) -> dict[str, Any]: + return compare_snapshots(self, other) + + +def create_snapshot(result: Mapping[str, Any]) -> AnalysisSnapshot: + """Create a deterministic compact state snapshot from an analysis result.""" + state = _state_payload(result) + return AnalysisSnapshot({ + "snapshot_schema_version": SNAPSHOT_SCHEMA_VERSION, + "created_at": datetime.now(timezone.utc).isoformat(), + "source": { + "dataset_id": result.get("dataset_id"), + "filename": result.get("filename"), + }, + "fingerprint": _fingerprint(state), + "state": state, + }) + + +def load_snapshot(path: str | Path) -> AnalysisSnapshot: + """Load and validate a FrameVitals snapshot from JSON.""" + source = Path(path) + if not source.exists(): + raise FileNotFoundError(f"Snapshot not found: {source}") + try: + payload = json.loads(source.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Snapshot is not valid JSON: {source}") from exc + + if not isinstance(payload, dict): + raise ValueError("Snapshot JSON must contain an object.") + if payload.get("snapshot_schema_version") != SNAPSHOT_SCHEMA_VERSION: + raise ValueError( + "Unsupported FrameVitals snapshot schema version: " + f"{payload.get('snapshot_schema_version')!r}" + ) + if not isinstance(payload.get("state"), dict): + raise ValueError("Snapshot is missing a valid state object.") + _created_at(payload) + return AnalysisSnapshot(payload) + + +def compare_snapshots( + reference: Mapping[str, Any], + current: Mapping[str, Any], +) -> dict[str, Any]: + """Compare compact analysis snapshots without requiring raw datasets.""" + ref_state = _as_mapping(reference.get("state")) + cur_state = _as_mapping(current.get("state")) + ref_dataset = _as_mapping(ref_state.get("dataset")) + cur_dataset = _as_mapping(cur_state.get("dataset")) + + ref_dtypes = dict(_as_mapping(ref_dataset.get("dtypes"))) + cur_dtypes = dict(_as_mapping(cur_dataset.get("dtypes"))) + ref_columns = set(ref_dtypes) + cur_columns = set(cur_dtypes) + + type_changes = { + column: {"reference": ref_dtypes[column], "current": cur_dtypes[column]} + for column in sorted(ref_columns & cur_columns) + if ref_dtypes[column] != cur_dtypes[column] + } + + ref_missing = dict(_as_mapping(ref_dataset.get("missing_percent"))) + cur_missing = dict(_as_mapping(cur_dataset.get("missing_percent"))) + missing_changes: dict[str, dict[str, float]] = {} + for column in sorted(set(ref_missing) & set(cur_missing)): + before = _number(ref_missing[column]) + after = _number(cur_missing[column]) + if before is None or after is None or before == after: + continue + missing_changes[column] = { + "reference": round(before, 4), + "current": round(after, 4), + "delta": round(after - before, 4), + } + + ref_health = _number(_as_mapping(ref_state.get("health")).get("overall_score")) + cur_health = _number(_as_mapping(cur_state.get("health")).get("overall_score")) + ref_ml = _number(_as_mapping(ref_state.get("ml_readiness")).get("score")) + cur_ml = _number(_as_mapping(cur_state.get("ml_readiness")).get("score")) + + ref_findings = set(ref_state.get("finding_codes", []) or []) + cur_findings = set(cur_state.get("finding_codes", []) or []) + + changed = bool(reference.get("fingerprint") != current.get("fingerprint")) + return { + "changed": changed, + "reference_fingerprint": reference.get("fingerprint"), + "current_fingerprint": current.get("fingerprint"), + "schema": { + "added_columns": sorted(cur_columns - ref_columns), + "removed_columns": sorted(ref_columns - cur_columns), + "type_changes": type_changes, + }, + "missingness_changes": missing_changes, + "health_delta": ( + round(cur_health - ref_health, 4) + if ref_health is not None and cur_health is not None + else None + ), + "ml_readiness_delta": ( + round(cur_ml - ref_ml, 4) + if ref_ml is not None and cur_ml is not None + else None + ), + "findings": { + "new": sorted(cur_findings - ref_findings), + "resolved": sorted(ref_findings - cur_findings), + }, + } + + +class SnapshotHistory: + """Filesystem-backed history of compact FrameVitals snapshots. + + The history store never writes raw datasets. It persists only the compact + snapshot representation and provides lightweight timeline/latest/diff + helpers for local monitoring and CI workflows. + """ + + def __init__(self, directory: str | Path = ".framevitals/history") -> None: + self.directory = Path(directory) + + def __len__(self) -> int: + return len(self.paths()) + + def paths(self) -> list[Path]: + """Return snapshot files in chronological filename order.""" + if not self.directory.exists(): + return [] + return sorted(path for path in self.directory.glob("*.json") if path.is_file()) + + def snapshots(self) -> list[AnalysisSnapshot]: + """Load all valid snapshots ordered by their created-at timestamp.""" + items = [load_snapshot(path) for path in self.paths()] + items.sort(key=_created_at) + return items + + def add( + self, + result_or_snapshot: Mapping[str, Any], + *, + label: str | None = None, + ) -> Path: + """Persist an analysis result or an existing snapshot and return its path.""" + if result_or_snapshot.get("snapshot_schema_version") is not None: + if result_or_snapshot.get("snapshot_schema_version") != SNAPSHOT_SCHEMA_VERSION: + raise ValueError( + "Unsupported FrameVitals snapshot schema version: " + f"{result_or_snapshot.get('snapshot_schema_version')!r}" + ) + if not isinstance(result_or_snapshot.get("state"), Mapping): + raise ValueError("Snapshot is missing a valid state object.") + snapshot = AnalysisSnapshot(dict(result_or_snapshot)) + created_at = _created_at(snapshot) + else: + snapshot = create_snapshot(result_or_snapshot) + created_at = _created_at(snapshot) + + self.directory.mkdir(parents=True, exist_ok=True) + timestamp = created_at.strftime("%Y%m%dT%H%M%S.%fZ") + fingerprint = str(snapshot.get("fingerprint") or "unknown")[:12] + label_part = _safe_label(label) + filename = "_".join( + part for part in (timestamp, label_part, fingerprint) if part + ) + ".json" + path = self.directory / filename + snapshot.to_json(path) + return path + + def latest(self) -> AnalysisSnapshot | None: + """Return the newest snapshot or ``None`` for an empty history.""" + items = self.snapshots() + return items[-1] if items else None + + def previous(self) -> AnalysisSnapshot | None: + """Return the snapshot immediately before the latest one, if available.""" + items = self.snapshots() + return items[-2] if len(items) >= 2 else None + + def compare_latest(self) -> dict[str, Any]: + """Compare the two newest snapshots.""" + items = self.snapshots() + if len(items) < 2: + raise ValueError("Snapshot history needs at least two entries to compare.") + return compare_snapshots(items[-2], items[-1]) + + def timeline(self) -> list[dict[str, Any]]: + """Return compact chronological monitoring points for charts or logs.""" + rows: list[dict[str, Any]] = [] + for snapshot in self.snapshots(): + state = _as_mapping(snapshot.get("state")) + dataset = _as_mapping(state.get("dataset")) + health = _as_mapping(state.get("health")) + ml = _as_mapping(state.get("ml_readiness")) + findings = state.get("finding_codes", []) + if not isinstance(findings, list): + findings = [] + rows.append({ + "created_at": snapshot.get("created_at"), + "fingerprint": snapshot.get("fingerprint"), + "filename": _as_mapping(snapshot.get("source")).get("filename"), + "shape": dict(_as_mapping(dataset.get("shape"))), + "health_score": _number(health.get("overall_score")), + "ml_readiness_score": _number(ml.get("score")), + "finding_count": len(findings), + }) + return rows diff --git a/src/framevitals/sources.py b/src/framevitals/sources.py new file mode 100644 index 0000000..c9bdffa --- /dev/null +++ b/src/framevitals/sources.py @@ -0,0 +1,480 @@ +"""Dataset source abstractions for FrameVitals. + +Sources expose cheap metadata before analysis. Streaming-capable sources can +additionally yield bounded record batches, allowing focused operations to avoid +materializing the complete dataset in pandas. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +import pandas as pd + +from framevitals.loader import load_dataset + + +@dataclass(frozen=True, slots=True) +class DatasetMetadata: + """Cheap source metadata available before deep analysis.""" + + name: str + kind: str + format: str + rows: int | None + columns: int | None + size_bytes: int | None + materialized: bool + supports_projection: bool + supports_streaming: bool + + def to_dict(self) -> dict: + return asdict(self) + + +@runtime_checkable +class DatasetSource(Protocol): + """Minimal contract implemented by FrameVitals data sources.""" + + def inspect(self) -> DatasetMetadata: ... + + def load(self) -> pd.DataFrame: ... + + +@runtime_checkable +class StreamingDatasetSource(DatasetSource, Protocol): + """Optional extension for sources that can yield bounded record batches.""" + + def iter_batches( + self, + *, + batch_size: int = 65_536, + columns: Sequence[str] | None = None, + ) -> Iterator[Any]: ... + + +@dataclass(slots=True) +class PandasSource: + dataframe: pd.DataFrame + name: str = "" + + def inspect(self) -> DatasetMetadata: + rows, columns = self.dataframe.shape + size_bytes = int(self.dataframe.memory_usage(index=True, deep=True).sum()) + return DatasetMetadata( + name=self.name, + kind="memory", + format="pandas", + rows=int(rows), + columns=int(columns), + size_bytes=size_bytes, + materialized=True, + supports_projection=True, + supports_streaming=False, + ) + + def load(self) -> pd.DataFrame: + if self.dataframe.empty: + raise ValueError("Dataset DataFrame is empty.") + return self.dataframe.copy() + + +def _normalize_arrow_table_types(table: Any) -> Any: + """Cast Arrow view types to compute-compatible equivalents without pandas.""" + try: + import pyarrow as pa + except ImportError: + return table + + fields = [] + changed = False + for field in table.schema: + target_type = field.type + if pa.types.is_string_view(target_type): + target_type = pa.string() + elif pa.types.is_binary_view(target_type): + target_type = pa.binary() + + changed = changed or target_type != field.type + fields.append( + pa.field( + field.name, + target_type, + nullable=field.nullable, + metadata=field.metadata, + ) + ) + + if not changed: + return table + + target_schema = pa.schema(fields, metadata=table.schema.metadata) + return table.cast(target_schema) + + +@dataclass(slots=True) +class ArrowTableSource: + """Projection-aware in-memory Arrow source without eager pandas conversion.""" + + table: Any + name: str = "" + + def __post_init__(self) -> None: + self.table = _normalize_arrow_table_types(self.table) + + def inspect(self) -> DatasetMetadata: + return DatasetMetadata( + name=self.name, + kind="memory", + format="arrow", + rows=int(self.table.num_rows), + columns=int(self.table.num_columns), + size_bytes=int(self.table.nbytes), + materialized=True, + supports_projection=True, + supports_streaming=True, + ) + + def schema(self): + """Return the native Arrow schema without converting row data.""" + return self.table.schema + + def iter_batches( + self, + *, + batch_size: int = 65_536, + columns: Sequence[str] | None = None, + ) -> Iterator[Any]: + if batch_size < 1: + raise ValueError("batch_size must be at least 1.") + projected = self.table + if columns is not None: + projected = projected.select(list(columns)) + yield from projected.to_batches(max_chunksize=int(batch_size)) + + def load(self) -> pd.DataFrame: + dataframe = self.table.to_pandas() + if dataframe.empty: + raise ValueError(f"Dataset is empty: {self.name}") + return dataframe + + +@dataclass(slots=True) +class FileSource: + path: Path + + def inspect(self) -> DatasetMetadata: + _validate_file_path(self.path) + suffix = self.path.suffix.lower().lstrip(".") or "unknown" + return DatasetMetadata( + name=self.path.name, + kind="file", + format=suffix, + rows=None, + columns=None, + size_bytes=int(self.path.stat().st_size), + materialized=False, + supports_projection=False, + supports_streaming=False, + ) + + def load(self) -> pd.DataFrame: + dataframe = load_dataset(self.path) + if dataframe.empty: + raise ValueError(f"Dataset is empty: {self.path}") + return dataframe + + +@dataclass(slots=True) +class DelimitedTextSource(FileSource): + """CSV/TSV file source with optional Arrow streaming acceleration. + + This remains a :class:`FileSource` for compatibility. Without the Arrow + extra it advertises the same non-streaming behaviour as a normal file + source and falls back to the existing pandas loader. With Arrow it performs + one bounded-memory metadata scan to obtain an exact row count, caches that + metadata, and then yields projected Arrow record batches. + """ + + delimiter: str = "," + _metadata_cache: DatasetMetadata | None = field(default=None, init=False, repr=False) + _schema_cache: Any = field(default=None, init=False, repr=False) + _arrow_compatible: bool | None = field(default=None, init=False, repr=False) + + @property + def format(self) -> str: + return "tsv" if self.delimiter == "\t" else "csv" + + def _pyarrow_csv(self): + try: + import pyarrow.csv as pacsv + except ImportError: + return None + return pacsv + + def _arrow_reader(self, *, columns: Sequence[str] | None = None): + pacsv = self._pyarrow_csv() + if pacsv is None: + raise ImportError( + "CSV/TSV streaming requires the optional Arrow capability. " + 'Install it with: pip install "framevitals[arrow]"' + ) + + read_options = pacsv.ReadOptions(use_threads=True) + parse_options = pacsv.ParseOptions(delimiter=self.delimiter) + convert_options = pacsv.ConvertOptions( + include_columns=list(columns) if columns is not None else None, + ) + return pacsv.open_csv( + str(self.path), + read_options=read_options, + parse_options=parse_options, + convert_options=convert_options, + ) + + def inspect(self) -> DatasetMetadata: + _validate_file_path(self.path) + if self._metadata_cache is not None: + return self._metadata_cache + + if self._pyarrow_csv() is None: + self._arrow_compatible = False + # Avoid zero-argument super() here: @dataclass(slots=True) may + # replace the class object, which breaks the implicit __class__ + # cell on Python versions where this fallback path is exercised. + self._metadata_cache = FileSource.inspect(self) + return self._metadata_cache + + try: + reader = self._arrow_reader() + schema = reader.schema + rows = 0 + while True: + try: + batch = reader.read_next_batch() + except StopIteration: + break + rows += int(batch.num_rows) + except Exception: + # Preserve existing CSV/TSV compatibility if Arrow cannot parse a + # file that the pandas loader may still understand. + self._arrow_compatible = False + self._metadata_cache = FileSource.inspect(self) + return self._metadata_cache + + self._arrow_compatible = True + self._schema_cache = schema + self._metadata_cache = DatasetMetadata( + name=self.path.name, + kind="file", + format=self.format, + rows=int(rows), + columns=int(len(schema)), + size_bytes=int(self.path.stat().st_size), + materialized=False, + supports_projection=True, + supports_streaming=True, + ) + return self._metadata_cache + + def schema(self): + """Return the inferred Arrow schema without materializing row data.""" + metadata = self.inspect() + if not metadata.supports_streaming: + raise TypeError( + f"{self.format.upper()} source is not Arrow-streamable: {self.path}" + ) + if self._schema_cache is None: + self._schema_cache = self._arrow_reader().schema + return self._schema_cache + + def iter_batches( + self, + *, + batch_size: int = 65_536, + columns: Sequence[str] | None = None, + ) -> Iterator[Any]: + if batch_size < 1: + raise ValueError("batch_size must be at least 1.") + metadata = self.inspect() + if not metadata.supports_streaming: + raise TypeError( + f"{self.format.upper()} source cannot stream without a compatible Arrow reader." + ) + + reader = self._arrow_reader(columns=columns) + while True: + try: + batch = reader.read_next_batch() + except StopIteration: + break + for offset in range(0, int(batch.num_rows), int(batch_size)): + yield batch.slice(offset, min(int(batch_size), int(batch.num_rows) - offset)) + + def load(self) -> pd.DataFrame: + return FileSource.load(self) + + +@dataclass(slots=True) +class ParquetSource: + """Projection-aware, streaming Parquet source backed by optional PyArrow. + + A single source object owns one ``ParquetFile`` instance plus cached schema + and metadata. Ultra-wide files can have expensive footer/schema parsing, so + repeatedly reopening the same file inside ``inspect()``, ``schema()`` and + ``iter_batches()`` wastes work without improving correctness. + """ + + path: Path + _parquet_file_cache: Any = field(default=None, init=False, repr=False) + _metadata_cache: DatasetMetadata | None = field(default=None, init=False, repr=False) + _schema_cache: Any = field(default=None, init=False, repr=False) + + def _parquet_file(self): + if self._parquet_file_cache is not None: + return self._parquet_file_cache + + _validate_file_path(self.path) + try: + import pyarrow.parquet as pq + except ImportError as exc: + raise ImportError( + "Parquet streaming requires the optional Arrow capability. " + 'Install it with: pip install "framevitals[arrow]"' + ) from exc + self._parquet_file_cache = pq.ParquetFile(self.path) + return self._parquet_file_cache + + def inspect(self) -> DatasetMetadata: + if self._metadata_cache is not None: + return self._metadata_cache + + parquet_file = self._parquet_file() + metadata = parquet_file.metadata + self._metadata_cache = DatasetMetadata( + name=self.path.name, + kind="file", + format="parquet", + rows=int(metadata.num_rows), + columns=int(metadata.num_columns), + size_bytes=int(self.path.stat().st_size), + materialized=False, + supports_projection=True, + supports_streaming=True, + ) + return self._metadata_cache + + def schema(self): + """Return the cached Arrow schema without reading row data.""" + if self._schema_cache is None: + self._schema_cache = self._parquet_file().schema_arrow + return self._schema_cache + + def iter_batches( + self, + *, + batch_size: int = 65_536, + columns: Sequence[str] | None = None, + ) -> Iterator[Any]: + if batch_size < 1: + raise ValueError("batch_size must be at least 1.") + yield from self._parquet_file().iter_batches( + batch_size=int(batch_size), + columns=list(columns) if columns is not None else None, + use_threads=True, + ) + + def load(self) -> pd.DataFrame: + table = self._parquet_file().read(use_threads=True) + dataframe = table.to_pandas() + if dataframe.empty: + raise ValueError(f"Dataset is empty: {self.path}") + return dataframe + + +def _validate_file_path(path: Path) -> None: + if not path.exists(): + raise FileNotFoundError(f"Dataset not found: {path}") + if not path.is_file(): + raise ValueError(f"Expected a dataset file, got: {path}") + + +def _arrow_memory_source(data: Any) -> ArrowTableSource | None: + """Recognize Arrow containers/producers without importing Arrow at package import.""" + try: + import pyarrow as pa + except ImportError: + return None + + if isinstance(data, pa.Table): + return ArrowTableSource(data) + if isinstance(data, pa.RecordBatch): + return ArrowTableSource( + pa.Table.from_batches([data]), + name="", + ) + if isinstance(data, pa.RecordBatchReader): + raise TypeError( + "Arrow RecordBatchReader inputs do not expose a cheap exact row count. " + "Pass a PyArrow Table/RecordBatch or another materialized Arrow-compatible " + "table object instead." + ) + + arrow_stream = getattr(data, "__arrow_c_stream__", None) + if callable(arrow_stream): + return ArrowTableSource( + pa.table(data), + name="", + ) + return None + + +def _duckdb_relation_source(data: Any): + """Recognize lazy DuckDB relations before generic Arrow conversion.""" + data_type = type(data) + if data_type.__name__ != "DuckDBPyRelation": + return None + if not data_type.__module__.startswith(("duckdb", "_duckdb")): + return None + + from framevitals.duckdb_source import resolve_duckdb_source + + return resolve_duckdb_source(data) + + +def resolve_source(data: Any) -> DatasetSource: + """Normalize supported user inputs into a DatasetSource implementation.""" + if isinstance(data, pd.DataFrame): + return PandasSource(data) + if isinstance(data, (str, Path)): + path = Path(data) + suffix = path.suffix.lower() + if suffix == ".parquet": + return ParquetSource(path) + if suffix == ".csv": + return DelimitedTextSource(path, delimiter=",") + if suffix == ".tsv": + return DelimitedTextSource(path, delimiter="\t") + return FileSource(path) + + duckdb_source = _duckdb_relation_source(data) + if duckdb_source is not None: + return duckdb_source + + arrow_source = _arrow_memory_source(data) + if arrow_source is not None: + return arrow_source + if isinstance(data, DatasetSource): + return data + raise TypeError( + "data must be a pandas DataFrame, Arrow-compatible table, DuckDB relation, " + "dataset path, or DatasetSource." + ) + + +def inspect_source(data: Any) -> dict[str, Any]: + """Return source metadata/capabilities without running analysis diagnostics.""" + return resolve_source(data).inspect().to_dict() diff --git a/src/framevitals/stream_change.py b/src/framevitals/stream_change.py new file mode 100644 index 0000000..ef1ad36 --- /dev/null +++ b/src/framevitals/stream_change.py @@ -0,0 +1,174 @@ +"""Low-overhead change detection for streaming and ordered numeric data. + +The detector consumes aggregate observations rather than every raw cell. That +keeps Python overhead proportional to ``columns * windows`` while still +surfacing sustained mean shifts in wide data and ordered time series. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import math + +import numpy as np +import pandas as pd + + +@dataclass(slots=True) +class PageHinkleyMeanShift: + """Scale-adaptive Page-Hinkley/CUSUM detector for sequential batch means.""" + + threshold: float = 8.0 + delta: float = 0.5 + min_updates: int = 8 + alpha: float = 0.995 + count: int = 0 + mean: float = 0.0 + m2: float = 0.0 + cumulative_up: float = 0.0 + cumulative_down: float = 0.0 + max_score: float = 0.0 + detected: bool = False + direction: str | None = None + detected_at: int | None = None + + def __post_init__(self) -> None: + if self.threshold <= 0: + raise ValueError("threshold must be positive.") + if self.delta < 0: + raise ValueError("delta must be non-negative.") + if self.min_updates < 3: + raise ValueError("min_updates must be at least 3.") + if not 0 < self.alpha <= 1: + raise ValueError("alpha must be in (0, 1].") + + @property + def variance(self) -> float | None: + if self.count < 2: + return None + return self.m2 / (self.count - 1) + + @property + def std(self) -> float | None: + variance = self.variance + if variance is None or variance <= 0: + return None + return math.sqrt(variance) + + def update(self, value: float | int | None) -> bool: + """Update with the next aggregate mean and return current detection state.""" + if value is None: + return self.detected + x = float(value) + if not math.isfinite(x): + return self.detected + + previous_mean = self.mean + previous_std = self.std + self.count += 1 + + if self.count == 1: + self.mean = x + return self.detected + + difference = x - self.mean + self.mean += difference / self.count + self.m2 += difference * (x - self.mean) + + if self.count < self.min_updates: + return self.detected + + scale = previous_std if previous_std is not None and previous_std > 1e-12 else None + if scale is None: + scale = max(abs(previous_mean) * 1e-6, 1e-9) + + # With standardized residuals, a 0.5-sigma allowance gives a much more + # useful false-positive tradeoff than a tiny raw-unit delta. The one-sided + # reset also prevents stationary random walks from accumulating forever. + residual = float(np.clip((x - previous_mean) / scale, -8.0, 8.0)) + self.cumulative_up = max( + 0.0, + self.alpha * self.cumulative_up + residual - self.delta, + ) + self.cumulative_down = max( + 0.0, + self.alpha * self.cumulative_down - residual - self.delta, + ) + + score = max(self.cumulative_up, self.cumulative_down) + self.max_score = max(self.max_score, float(score)) + + if not self.detected and score >= self.threshold: + self.detected = True + self.direction = "up" if self.cumulative_up >= self.cumulative_down else "down" + self.detected_at = self.count + return self.detected + + def snapshot(self) -> dict[str, Any]: + return { + "method": "page_hinkley_batch_means", + "updates": int(self.count), + "detected": bool(self.detected), + "direction": self.direction, + "detected_at_batch": self.detected_at, + "max_score": round(float(self.max_score), 6), + "threshold": float(self.threshold), + "delta": float(self.delta), + "minimum_batches": int(self.min_updates), + "sufficient_batches": bool(self.count >= self.min_updates), + } + + +def scan_ordered_mean_shift( + series: pd.Series, + *, + windows: int = 24, + threshold: float = 8.0, + min_updates: int = 8, +) -> dict[str, Any]: + """Detect sustained mean changes using bounded contiguous window summaries.""" + if windows < min_updates: + raise ValueError("windows must be at least min_updates.") + + numeric = pd.to_numeric(series, errors="coerce").replace([np.inf, -np.inf], np.nan) + values = numeric.to_numpy(dtype=np.float64, na_value=np.nan) + if values.size < min_updates * 4: + return { + "available": False, + "reason": "Too few ordered observations for bounded change detection.", + "method": "page_hinkley_window_means", + "observations": int(values.size), + } + + effective_windows = min(int(windows), max(min_updates, values.size // 4)) + chunks = np.array_split(values, effective_windows) + detector = PageHinkleyMeanShift( + threshold=threshold, + min_updates=min_updates, + ) + window_means: list[float | None] = [] + for chunk in chunks: + finite = chunk[np.isfinite(chunk)] + mean = float(np.mean(finite)) if finite.size else None + window_means.append(mean) + detector.update(mean) + + snapshot = detector.snapshot() + valid_means = [value for value in window_means if value is not None] + return { + "available": bool(valid_means), + "method": "page_hinkley_window_means", + "observations": int(values.size), + "windows": int(effective_windows), + "window_means_preview": [ + None if value is None else round(float(value), 6) + for value in window_means[-12:] + ], + "detected": bool(snapshot["detected"]), + "direction": snapshot["direction"], + "detected_at_window": snapshot["detected_at_batch"], + "max_score": snapshot["max_score"], + "threshold": snapshot["threshold"], + } diff --git a/src/framevitals/streaming_bounded_pipeline.py b/src/framevitals/streaming_bounded_pipeline.py new file mode 100644 index 0000000..eb6a517 --- /dev/null +++ b/src/framevitals/streaming_bounded_pipeline.py @@ -0,0 +1,301 @@ +"""Bounded module scheduler for streaming analyses. + +Streaming sources already own profile, role, health, readiness, quality, and +source-shape state. Re-entering the materialized pipeline on the retained sample +would recompute those facts and then throw them away. This scheduler runs only +modules that genuinely require row-level values while consuming the source-level +execution budget chosen by the streaming planner. +""" + +from __future__ import annotations + +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Callable + +import pandas as pd + +from framevitals.advanced_indicators import calculate_advanced_indicators +from framevitals.budgeted_analysis import ( + run_budgeted_anomalies, + run_budgeted_deep_statistics, + run_budgeted_time_series, +) +from framevitals.config import VALID_MODULES +from framevitals.execution import ExecutionBudget, derive_execution_budget +from framevitals.model_leaderboard import run_model_leaderboard +from framevitals.pipeline import _result_status, _safe_call, _skipped_module +from framevitals.target_intelligence import run_target_intelligence +from framevitals.text_profile import profile_text_columns + + +def run_streaming_bounded_modules( + dataframe: pd.DataFrame, + *, + dataset_id: str, + original_filename: str, + analysis_mode: str, + target_column: str | None, + parallel_workers: int, + source_budget: ExecutionBudget, + column_roles: dict[str, Any], + skip_ai: bool, + disabled_modules: set[str] | tuple[str, ...] | list[str] | None = None, +) -> dict[str, Any]: + """Run only row-dependent modules on the bounded streaming sample. + + Statistical adapters receive ``source_budget`` so ultra-wide source limits + remain authoritative even though the retained sample is much narrower. The + scheduler's concurrency limit, however, is derived from the bounded sample + because that is the memory footprint actually resident during parallel work. + """ + overall_start = time.perf_counter() + timings_ms: dict[str, Any] = {} + disabled = set(disabled_modules or ()) + unknown = sorted(disabled - VALID_MODULES) + if unknown: + raise ValueError("Unknown disabled module(s): " + ", ".join(unknown)) + if dataframe.empty: + raise ValueError("Bounded streaming sample is empty.") + if target_column is not None and target_column not in dataframe.columns: + raise ValueError(f"Target column not found: {target_column}") + + module_status: dict[str, str] = { + name: "pending" for name in sorted(VALID_MODULES) + } + + def module_enabled(name: str) -> bool: + return name not in disabled + + t0 = time.perf_counter() + advanced = calculate_advanced_indicators(dataframe) + timings_ms["advanced"] = (time.perf_counter() - t0) * 1000 + + phase3_modules: list[tuple[str, str, Callable[[], Any]]] = [ + ( + "deep_statistics", + "deep_statistics_v2", + lambda: run_budgeted_deep_statistics(dataframe, budget=source_budget), + ), + ( + "anomaly_detection", + "anomalies_v2", + lambda: run_budgeted_anomalies(dataframe, budget=source_budget), + ), + ( + "time_series", + "time_series", + lambda: run_budgeted_time_series( + dataframe, + budget=source_budget, + target_column=target_column, + ), + ), + ("text_profile", "text_profile", lambda: profile_text_columns(dataframe)), + ] + + phase3_results: dict[str, Any] = {} + bounded_parallel_budget = derive_execution_budget( + len(dataframe), + len(dataframe.columns), + mode=analysis_mode, + ) + phase3_worker_limit = max( + 1, + min( + int(parallel_workers), + int(bounded_parallel_budget.max_memory_heavy_parallelism), + ), + ) + + if analysis_mode in {"standard", "deep", "research"}: + tasks: list[tuple[str, str, Callable[[], Any]]] = [] + for module, result_key, fn in phase3_modules: + if module_enabled(module): + tasks.append((module, result_key, fn)) + module_status[module] = "scheduled" + else: + phase3_results[result_key] = _skipped_module(module) + module_status[module] = "disabled" + + per_task_ms: dict[str, float] = {} + if tasks: + phase3_start = time.perf_counter() + with ThreadPoolExecutor(max_workers=phase3_worker_limit) as executor: + futures = { + executor.submit(_safe_call, result_key, fn): (module, result_key) + for module, result_key, fn in tasks + } + for future in as_completed(futures): + module, result_key = futures[future] + name, value, elapsed = future.result() + phase3_results[result_key] = value + per_task_ms[name] = elapsed + module_status[module] = _result_status(value) + timings_ms["phase3_parallel_total"] = ( + time.perf_counter() - phase3_start + ) * 1000 + else: + timings_ms["phase3_parallel_total"] = 0.0 + timings_ms["phase3_tasks"] = per_task_ms + else: + timings_ms["phase3_parallel_total"] = 0.0 + timings_ms["phase3_tasks"] = {} + for module, _, _ in phase3_modules: + module_status[module] = "not_applicable" + + deep_statistics_v2 = phase3_results.get("deep_statistics_v2") + anomalies_v2 = phase3_results.get("anomalies_v2") + time_series_analysis = phase3_results.get("time_series") + text_profile = phase3_results.get("text_profile") + + target_intelligence = None + model_leaderboard = None + explainability = None + + if target_column: + if module_enabled("target_intelligence"): + t0 = time.perf_counter() + _, target_intelligence, _ = _safe_call( + "target_intelligence", + lambda: run_target_intelligence( + dataframe, + target_column=target_column, + column_roles=column_roles, + ), + ) + timings_ms["target_intelligence"] = (time.perf_counter() - t0) * 1000 + module_status["target_intelligence"] = _result_status(target_intelligence) + else: + target_intelligence = _skipped_module("target_intelligence") + timings_ms["target_intelligence"] = 0.0 + module_status["target_intelligence"] = "disabled" + else: + module_status["target_intelligence"] = "not_applicable" + + modeling_applicable = bool( + target_column and analysis_mode in {"standard", "deep", "research"} + ) + if modeling_applicable: + if module_enabled("modeling"): + t0 = time.perf_counter() + _, model_leaderboard, _ = _safe_call( + "model_leaderboard", + lambda: run_model_leaderboard(dataframe, target_column=target_column), + ) + timings_ms["model_leaderboard"] = (time.perf_counter() - t0) * 1000 + module_status["modeling"] = _result_status(model_leaderboard) + else: + model_leaderboard = _skipped_module("modeling") + timings_ms["model_leaderboard"] = 0.0 + module_status["modeling"] = "disabled" + else: + module_status["modeling"] = "not_applicable" + + winner_available = ( + isinstance(model_leaderboard, dict) + and model_leaderboard.get("available") + and model_leaderboard.get("winner") + ) + if winner_available: + if module_enabled("explainability"): + t0 = time.perf_counter() + + def run_explainability(): + from framevitals.explainability import explain_winner + + return explain_winner( + dataframe, + target_column=target_column, + leaderboard_result=model_leaderboard, + dataset_id=dataset_id, + ) + + _, explainability, _ = _safe_call("explainability", run_explainability) + timings_ms["explainability"] = (time.perf_counter() - t0) * 1000 + module_status["explainability"] = _result_status(explainability) + else: + explainability = _skipped_module("explainability") + timings_ms["explainability"] = 0.0 + module_status["explainability"] = "disabled" + elif not module_enabled("explainability"): + explainability = _skipped_module("explainability") + module_status["explainability"] = "disabled" + else: + module_status["explainability"] = "not_applicable" + + # Full AI generation consumes the final full-stream profile/signals and is + # therefore intentionally left to the outer streaming orchestrator. The + # public analyze API currently requests skip_ai=True on this path. + if not module_enabled("ai"): + ai_report = { + "source": "disabled", + "text": "AI report disabled by configuration.", + "deferred": False, + } + module_status["ai"] = "disabled" + elif skip_ai: + ai_report = { + "source": "skipped", + "text": "AI report skipped.", + "deferred": False, + } + module_status["ai"] = "skipped_by_caller" + else: + ai_env = os.environ.get( + "FRAMEVITALS_ANALYZE_AI", + os.environ.get("DATALENS_ANALYZE_AI", "0"), + ) + enabled = ai_env.strip().lower() in {"1", "true", "yes"} + ai_report = { + "source": "deferred", + "text": "", + "deferred": True, + "reason": ( + "Streaming AI interpretation is deferred until full-stream core " + "signals are assembled." + if enabled + else "AI analysis is not enabled by environment." + ), + } + module_status["ai"] = "deferred" + timings_ms["ai_report"] = 0.0 + + # Core streaming modules are intentionally not executed here. Their status + # is filled by the outer streaming pipeline after reuse. + for module in ("quality_diagnostics", "cleaning", "charts"): + if module in disabled: + module_status[module] = "disabled" + + timings_ms["total"] = (time.perf_counter() - overall_start) * 1000 + return { + "dataset_id": dataset_id, + "filename": original_filename, + "analysis_mode": analysis_mode, + "artifacts_enabled": False, + "execution": { + "disabled_modules": sorted(disabled), + "module_status": module_status, + "budget": source_budget.to_dict(), + "phase3_worker_limit": int(phase3_worker_limit), + "bounded_scheduler": { + "enabled": True, + "core_reprofiled": False, + "source_budget_scope": "source_shape", + "parallelism_budget_scope": "bounded_sample", + "sample_rows": int(len(dataframe)), + "sample_columns": int(len(dataframe.columns)), + }, + }, + "advanced": advanced, + "deep_statistics_v2": deep_statistics_v2, + "anomalies_v2": anomalies_v2, + "target_intelligence": target_intelligence, + "model_leaderboard": model_leaderboard, + "explainability": explainability, + "time_series": time_series_analysis, + "text_profile": text_profile, + "ai_report": ai_report, + "timings_ms": timings_ms, + } diff --git a/src/framevitals/streaming_exact_reuse.py b/src/framevitals/streaming_exact_reuse.py new file mode 100644 index 0000000..53aea5b --- /dev/null +++ b/src/framevitals/streaming_exact_reuse.py @@ -0,0 +1,107 @@ +"""Reuse full-stream sufficient statistics in bounded downstream analyses. + +The streaming profiler has already scanned every selected source row and owns +higher-quality sufficient statistics than any bounded diagnostic sample can +reconstruct. This module enforces FrameVitals' exact-once rule: downstream +analysis payloads may add sample-dependent diagnostics, but they must not replace +known full-stream moments with noisier sample estimates. +""" + +from __future__ import annotations + +from typing import Any + +from framevitals.deep_statistics_v2 import _classify_kurtosis, _classify_skew + + +_EXACT_NUMERIC_FIELD_MAP = { + "count": "count", + "mean": "mean", + "std": "std", + "min": "min", + "max": "max", + "skewness": "skewness", + "kurtosis": "kurtosis", +} + + +def reuse_streaming_exact_statistics(payload: dict[str, Any]) -> dict[str, Any]: + """Overlay full-stream numeric sufficient statistics onto deep diagnostics. + + Quantiles, outlier views, distribution fits, hypothesis tests, confidence + intervals, and bivariate tests remain bounded-sample diagnostics. Exact + count/mean/std/min/max and central-moment shape statistics are reused from + the already-completed full-stream profile pass. + """ + profile = payload.get("profile") + deep = payload.get("deep_statistics_v2") + if not isinstance(profile, dict) or not isinstance(deep, dict): + return payload + + deep_numeric = deep.get("numeric_statistics") + full_numeric = profile.get("numeric_summary") + if not isinstance(deep_numeric, dict) or not isinstance(full_numeric, dict): + return payload + + streaming = profile.get("streaming_metadata", {}) + backend = streaming.get("numeric_backend") + profile_metadata = profile.get("numeric_summary_metadata", {}) + method = profile_metadata.get("method") if isinstance(profile_metadata, dict) else None + + reused_columns = 0 + for column, sample_summary in deep_numeric.items(): + if not isinstance(sample_summary, dict): + continue + exact_summary = full_numeric.get(column) + if not isinstance(exact_summary, dict): + continue + + reused_fields: list[str] = [] + for target_field, source_field in _EXACT_NUMERIC_FIELD_MAP.items(): + if source_field not in exact_summary: + continue + sample_summary[target_field] = exact_summary[source_field] + reused_fields.append(target_field) + + if "skewness" in reused_fields: + sample_summary["skewness_label"] = _classify_skew( + sample_summary.get("skewness") + ) + if "kurtosis" in reused_fields: + sample_summary["kurtosis_label"] = _classify_kurtosis( + sample_summary.get("kurtosis") + ) + + if reused_fields: + reused_columns += 1 + sample_summary["summary_provenance"] = { + "scope": "full_stream", + "backend": backend, + "method": method, + "reused_exact_fields": reused_fields, + "sample_derived_fields": [ + "q1", + "median", + "q3", + "iqr", + "outliers", + "normality", + "distribution_fit", + "bootstrap_mean_ci", + "bootstrap_median_ci", + ], + } + + if reused_columns: + execution = deep.setdefault("execution", {}) + if isinstance(execution, dict): + execution["exact_once_reuse"] = { + "enabled": True, + "source": "streaming_profile", + "backend": backend, + "method": method, + "columns_reused": reused_columns, + "fields": list(_EXACT_NUMERIC_FIELD_MAP), + } + + return payload diff --git a/src/framevitals/streaming_pipeline.py b/src/framevitals/streaming_pipeline.py new file mode 100644 index 0000000..eb541a9 --- /dev/null +++ b/src/framevitals/streaming_pipeline.py @@ -0,0 +1,497 @@ +"""Full FrameVitals orchestration for streaming dataset sources. + +The streaming pipeline performs one full-source Arrow scan to build reusable +profile state, retains one bounded deterministic working sample, and runs only +row-dependent modules on that sample. Full-source facts are reused instead of +re-entering the materialized pandas pipeline, and every bounded module is +explicitly scoped so downstream consumers never mistake sample-derived +diagnostics for full-data execution. +""" + +from __future__ import annotations + +import time +from collections.abc import Iterable, Sequence +from typing import Any + +from framevitals.analysis_selector import select_analyses +from framevitals.column_roles import summarize_roles +from framevitals.config import VALID_MODULES +from framevitals.dataset_signals import detect_dataset_signals +from framevitals.execution import ( + derive_execution_budget, + derive_streaming_profile_column_limit, +) +from framevitals.health_score import calculate_health_score_from_profile_sample +from framevitals.ml_readiness import calculate_ml_readiness_from_profile +from framevitals.signal_engine import build_signals +from framevitals.sources import DatasetMetadata, StreamingDatasetSource +from framevitals.streaming_bounded_pipeline import run_streaming_bounded_modules +from framevitals.streaming_profile import build_streaming_profile +from framevitals.streaming_quality import run_streaming_quality_diagnostics +from framevitals.streaming_roles import infer_streaming_column_roles + + +_BOUNDED_RESULT_KEYS = ( + "advanced", + "deep_statistics_v2", + "anomalies_v2", + "target_intelligence", + "model_leaderboard", + "explainability", + "time_series", + "text_profile", +) + + +def _working_sample_rows(budget) -> int: + """Choose one reusable row sample large enough for every bounded module.""" + requested = max( + budget.quality_sample_rows, + budget.deep_statistics_sample_rows, + budget.anomaly_sample_rows, + budget.time_series_sample_rows, + budget.pair_sample_rows, + 1, + ) + return min(int(budget.rows), int(requested)) + + +def _evenly_spaced_names(names: Sequence[str], limit: int) -> list[str]: + total = len(names) + if total <= limit: + return list(names) + if limit <= 1: + return [str(names[0])] + selected = [ + str(names[(index * (total - 1)) // (limit - 1)]) + for index in range(limit) + ] + return list(dict.fromkeys(selected)) + + +def _streaming_profile_projection( + source: StreamingDatasetSource, + *, + source_columns: int, + limit: int, + target_column: str | None, +) -> list[str] | None: + """Choose a deterministic schema-wide projection for ultra-wide sources.""" + if source_columns <= limit: + return None + + schema_method = getattr(source, "schema", None) + if not callable(schema_method): + raise TypeError( + "Ultra-wide streaming analysis requires source.schema() so FrameVitals " + "can project columns instead of scanning the complete width." + ) + schema = schema_method() + names = [str(field.name) for field in schema] + if target_column is not None and target_column not in names: + raise ValueError(f"Target column not found: {target_column}") + + selected = _evenly_spaced_names(names, limit) + if target_column is not None and target_column not in selected: + selected[-1] = target_column + return list(dict.fromkeys(selected)) + + +class _ProjectedStreamingSource: + """Projection-enforcing view over a streaming source. + + The wrapped source is never asked for the complete ultra-wide width. This + keeps the reusable full-row profile scan bounded while preserving source + metadata in the outer pipeline. + """ + + def __init__(self, source: StreamingDatasetSource, columns: Sequence[str]): + self._source = source + self._columns = tuple(columns) + + def inspect(self) -> DatasetMetadata: + metadata = self._source.inspect() + return DatasetMetadata( + name=metadata.name, + kind=metadata.kind, + format=metadata.format, + rows=metadata.rows, + columns=len(self._columns), + size_bytes=metadata.size_bytes, + materialized=metadata.materialized, + supports_projection=True, + supports_streaming=True, + ) + + def schema(self): + schema_method = getattr(self._source, "schema", None) + if not callable(schema_method): + raise TypeError("Projected streaming sources require schema().") + schema = schema_method() + return [schema.field(name) for name in self._columns] + + def iter_batches( + self, + *, + batch_size: int = 65_536, + columns: Sequence[str] | None = None, + ): + requested = list(self._columns if columns is None else columns) + allowed = set(self._columns) + unknown = [column for column in requested if column not in allowed] + if unknown: + raise ValueError( + "Projected source requested columns outside its budget: " + + ", ".join(unknown[:5]) + ) + yield from self._source.iter_batches( + batch_size=batch_size, + columns=requested, + ) + + def load(self): + raise RuntimeError("Projected streaming source must never materialize fully.") + + +def _scope_bounded_result( + value: Any, + *, + source_rows: int, + sample_rows: int, + strategy: str, +) -> Any: + if not isinstance(value, dict): + return value + scoped = dict(value) + scoped["execution_scope"] = { + "scope": "bounded_row_sample" if sample_rows < source_rows else "full_source", + "full_materialization": False, + "source_rows": int(source_rows), + "sample_rows": int(sample_rows), + "sampled": bool(sample_rows < source_rows), + "strategy": strategy, + } + return scoped + + +def _streaming_cleaning_result( + *, + enabled_by_user: bool, + profile: dict[str, Any], + health: dict[str, Any], +) -> dict[str, Any]: + missing_count = sum( + int(value) + for value in profile.get("missing_counts", {}).values() + if value is not None + ) + duplicate_count = int(profile.get("duplicate_rows", 0) or 0) + if enabled_by_user: + reason = ( + "Full-source cleaning is not executed implicitly on the streaming analysis " + "path because applying sample-derived mutations would be unsafe. Run the " + "explicit cleaning workflow when a transformed dataset is required." + ) + status = "deferred_streaming" + else: + reason = "Disabled by configuration." + status = "disabled" + return { + "available": False, + "skipped": True, + "module": "cleaning", + "reason": reason, + "streaming_status": status, + "actions": [], + "before_health": health, + "after_health": health, + "output_path": None, + "missing_before": missing_count, + "missing_after": missing_count, + "duplicates_before": duplicate_count, + "duplicates_after": duplicate_count, + } + + +def run_streaming_analysis( + *, + source: StreamingDatasetSource, + dataset_id: str, + original_filename: str, + analysis_mode: str, + target_column: str | None, + parallel_workers: int, + skip_ai: bool, + disabled_modules: Iterable[str] | None = None, +) -> dict[str, Any]: + """Run a stable-shape full analysis without materializing the complete source.""" + overall_start = time.perf_counter() + metadata = source.inspect() + rows = int(metadata.rows or 0) + columns = int(metadata.columns or 0) + if rows < 1 or columns < 1: + raise ValueError(f"Dataset is empty or has no columns: {metadata.name}") + + user_disabled = set(disabled_modules or ()) + unknown = sorted(user_disabled - VALID_MODULES) + if unknown: + raise ValueError("Unknown disabled module(s): " + ", ".join(unknown)) + + budget = derive_execution_budget(rows, columns, mode=analysis_mode) + working_rows = _working_sample_rows(budget) + profile_column_limit = derive_streaming_profile_column_limit( + rows, + columns, + mode=analysis_mode, + ) + profile_columns = _streaming_profile_projection( + source, + source_columns=columns, + limit=profile_column_limit, + target_column=target_column, + ) + profile_source: StreamingDatasetSource = ( + _ProjectedStreamingSource(source, profile_columns) + if profile_columns is not None + else source + ) + + core_timings: dict[str, float] = {} + t0 = time.perf_counter() + profile, working_sample = build_streaming_profile( + profile_source, + sample_rows=working_rows, + return_sample=True, + ) + core_timings["streaming_profile"] = (time.perf_counter() - t0) * 1000 + + profiled_columns = int(len(working_sample.columns)) + column_sampled = profiled_columns < columns + profile["shape"] = {"rows": rows, "columns": columns} + profile["source_metadata"] = metadata.to_dict() + streaming_metadata = dict(profile.get("streaming_metadata", {})) + streaming_metadata.update({ + "source_columns": columns, + "profiled_columns": profiled_columns, + "column_sampled": column_sampled, + "column_limit": int(profile_column_limit), + "column_strategy": ( + "deterministic_schema_projection" if column_sampled else "full_schema" + ), + }) + profile["streaming_metadata"] = streaming_metadata + + if working_sample.empty: + raise ValueError(f"Streaming source produced no usable rows: {metadata.name}") + if target_column is not None and target_column not in working_sample.columns: + raise ValueError(f"Target column not found: {target_column}") + + t0 = time.perf_counter() + role_payload = infer_streaming_column_roles(working_sample, profile=profile) + column_roles = role_payload["columns"] + roles_summary = summarize_roles(column_roles) + core_timings["column_roles"] = (time.perf_counter() - t0) * 1000 + + t0 = time.perf_counter() + health = calculate_health_score_from_profile_sample(profile, working_sample) + core_timings["health"] = (time.perf_counter() - t0) * 1000 + + t0 = time.perf_counter() + ml_readiness = calculate_ml_readiness_from_profile(profile) + core_timings["ml_readiness"] = (time.perf_counter() - t0) * 1000 + + t0 = time.perf_counter() + quality_diagnostics = ( + run_streaming_quality_diagnostics( + working_sample, + profile=profile, + source_rows=rows, + source_columns=columns, + max_sample_rows=max(budget.quality_sample_rows, 10), + ) + if "quality_diagnostics" not in user_disabled + else { + "available": False, + "skipped": True, + "module": "quality_diagnostics", + "reason": "Disabled by configuration.", + } + ) + core_timings["quality_diagnostics"] = (time.perf_counter() - t0) * 1000 + + t0 = time.perf_counter() + dataset_signals = detect_dataset_signals( + working_sample, + profile, + column_roles=column_roles, + source_shape=(rows, columns), + ) + core_timings["dataset_signals"] = (time.perf_counter() - t0) * 1000 + + t0 = time.perf_counter() + analysis_selection = select_analyses( + signals=dataset_signals, + analysis_mode=analysis_mode, + target_column=target_column, + ) + analysis_selection["execution_modules"] = { + "disabled": sorted(user_disabled), + "enabled": sorted(VALID_MODULES - user_disabled), + } + analysis_selection["execution_budget"] = budget.to_dict() + analysis_selection["streaming"] = { + "enabled": True, + "full_materialization": False, + "working_sample_rows": int(len(working_sample)), + "source_rows": rows, + "source_columns": columns, + "profiled_columns": profiled_columns, + "column_sampled": column_sampled, + } + core_timings["analysis_selection"] = (time.perf_counter() - t0) * 1000 + + # Cleaning/charts are internally suppressed on the bounded sample. The + # streaming scheduler runs only modules that genuinely require row values; + # profile/roles/health/readiness/quality above are never recomputed. + internal_disabled = set(user_disabled) | {"cleaning", "charts"} + sample_payload = run_streaming_bounded_modules( + working_sample, + dataset_id=dataset_id, + original_filename=original_filename, + analysis_mode=analysis_mode, + target_column=target_column, + parallel_workers=parallel_workers, + source_budget=budget, + column_roles=column_roles, + skip_ai=skip_ai, + disabled_modules=tuple(sorted(internal_disabled)), + ) + + sample_rows = int(len(working_sample)) + sample_strategy = "streaming_stratified_jitter_global_rows" + for key in _BOUNDED_RESULT_KEYS: + sample_payload[key] = _scope_bounded_result( + sample_payload.get(key), + source_rows=rows, + sample_rows=sample_rows, + strategy=sample_strategy, + ) + + advanced = sample_payload.get("advanced") + t0 = time.perf_counter() + signals = build_signals(profile, health, ml_readiness, advanced) + core_timings["signals"] = (time.perf_counter() - t0) * 1000 + + cleaning = _streaming_cleaning_result( + enabled_by_user="cleaning" not in user_disabled, + profile=profile, + health=health, + ) + + execution = dict(sample_payload.get("execution", {})) + module_status = dict(execution.get("module_status", {})) + module_status["cleaning"] = ( + "disabled" if "cleaning" in user_disabled else "deferred_streaming" + ) + module_status["charts"] = ( + "disabled" if "charts" in user_disabled else "not_applicable" + ) + module_status["quality_diagnostics"] = ( + "disabled" if "quality_diagnostics" in user_disabled else "ran" + ) + + module_scope: dict[str, str] = {} + for module in ( + "deep_statistics", + "anomaly_detection", + "time_series", + "text_profile", + "target_intelligence", + "modeling", + "explainability", + ): + status = module_status.get(module) + if status in {"ran", "scheduled", "error"}: + module_scope[module] = ( + "bounded_row_sample" if sample_rows < rows else "full_source" + ) + profile_scope = ( + "full_rows_projected_columns" if column_sampled else "full_stream" + ) + module_scope.update({ + "profile": profile_scope, + "column_roles": ( + "full_rows_projected_columns_plus_bounded_semantic_sample" + if column_sampled + else "full_stream_plus_bounded_semantic_sample" + ), + "health": ( + "full_rows_projected_columns_plus_bounded_outlier_sample" + if column_sampled + else "full_stream_plus_bounded_outlier_sample" + ), + "ml_readiness": ( + "full_rows_projected_columns_profile" + if column_sampled + else "full_stream_profile" + ), + "quality_diagnostics": ( + "disabled" + if "quality_diagnostics" in user_disabled + else ( + "full_rows_projected_columns_plus_bounded_value_sample" + if column_sampled + else "full_stream_plus_bounded_value_sample" + ) + ), + "cleaning": "deferred_streaming" if "cleaning" not in user_disabled else "disabled", + "charts": "not_applicable" if "charts" not in user_disabled else "disabled", + }) + + execution.update({ + "disabled_modules": sorted(user_disabled), + "internally_disabled_modules": sorted(internal_disabled - user_disabled), + "module_status": module_status, + "module_scope": module_scope, + "budget": budget.to_dict(), + "streaming": { + "enabled": True, + "source": metadata.to_dict(), + "full_materialization": False, + "source_rows": rows, + "source_columns": columns, + "profiled_columns": profiled_columns, + "column_sampled": column_sampled, + "column_limit": int(profile_column_limit), + "column_strategy": ( + "deterministic_schema_projection" if column_sampled else "full_schema" + ), + "working_sample_rows": sample_rows, + "working_sample_strategy": sample_strategy, + "single_full_source_profile_scan": True, + }, + }) + + timings = dict(sample_payload.get("timings_ms", {})) + timings.update(core_timings) + timings["total"] = (time.perf_counter() - overall_start) * 1000 + + sample_payload.update({ + "filename": original_filename, + "artifacts_enabled": False, + "execution": execution, + "profile": profile, + "column_roles": column_roles, + "roles_summary": roles_summary, + "dataset_signals": dataset_signals, + "analysis_selection": analysis_selection, + "health": health, + "signals": signals, + "ml_readiness": ml_readiness, + "quality_diagnostics": quality_diagnostics, + "cleaning": cleaning, + "charts": [], + "timings_ms": timings, + }) + return sample_payload diff --git a/src/framevitals/streaming_profile.py b/src/framevitals/streaming_profile.py new file mode 100644 index 0000000..5b77746 --- /dev/null +++ b/src/framevitals/streaming_profile.py @@ -0,0 +1,737 @@ +"""Streaming profile construction for Arrow-capable dataset sources. + +The streaming path keeps full-row work to mergeable column state and retains +only a bounded, deterministic stratified-jitter row sample for analyses that +still require row relationships (duplicate estimation and correlation). Native +builds consume Arrow RecordBatches directly for fused numeric profiling and +Arrow UTF-8 buffers for full-file categorical sketches when those kernels are +available. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import asdict +from typing import Any + +import numpy as np +import pandas as pd + +from framevitals.analysis_state import NumericColumnState +from framevitals.backends import ( + create_arrow_batch_profile_accumulator, + create_numeric_accumulator, + create_string_accumulator, + numeric_state, + resolve_numeric_backend, +) +from framevitals.execution import _deterministic_stratified_positions +from framevitals.profiler import _bounded_correlations, series_to_dict +from framevitals.sources import StreamingDatasetSource +from framevitals.streaming_sketches import ( + NumpyLogQuantileSketch, + PYTHON_NUMERIC_SKETCH_CELL_BUDGET, + should_use_full_stream_numpy_sketch, +) + + +STREAM_BATCH_SIZE = 65_536 +STREAM_SAMPLE_ROWS = 50_000 +STREAM_BATCH_CELL_BUDGET = 32_000_000 +STREAM_SAMPLE_CELL_BUDGET = 6_000_000 + + +def _width_aware_row_limit( + requested_rows: int, + column_count: int, + *, + cell_budget: int, +) -> int: + """Clamp a row budget so rows * columns stays within a bounded cell budget.""" + if requested_rows < 1: + raise ValueError("requested_rows must be at least 1.") + if cell_budget < 1: + raise ValueError("cell_budget must be at least 1.") + + width = max(int(column_count), 1) + width_limited_rows = max(int(cell_budget) // width, 1) + return max(1, min(int(requested_rows), width_limited_rows)) + + +def _require_pyarrow(): + try: + import pyarrow as pa + except ImportError as exc: + raise ImportError( + "Streaming Arrow profiling requires the optional Arrow capability. " + 'Install it with: pip install "framevitals[arrow]"' + ) from exc + return pa + + +def _column_groups(schema) -> tuple[list[str], list[str], list[str]]: + pa = _require_pyarrow() + numeric: list[str] = [] + categorical: list[str] = [] + dates: list[str] = [] + + for field in schema: + dtype = field.type + if pa.types.is_integer(dtype) or pa.types.is_floating(dtype): + numeric.append(field.name) + elif ( + pa.types.is_string(dtype) + or pa.types.is_large_string(dtype) + or pa.types.is_dictionary(dtype) + or pa.types.is_boolean(dtype) + or pa.types.is_binary(dtype) + or pa.types.is_large_binary(dtype) + ): + categorical.append(field.name) + elif ( + pa.types.is_timestamp(dtype) + or pa.types.is_date32(dtype) + or pa.types.is_date64(dtype) + or pa.types.is_time32(dtype) + or pa.types.is_time64(dtype) + or pa.types.is_duration(dtype) + ): + dates.append(field.name) + + return numeric, categorical, dates + + +def _string_columns(schema) -> list[str]: + pa = _require_pyarrow() + return [ + field.name + for field in schema + if pa.types.is_string(field.type) or pa.types.is_large_string(field.type) + ] + + +def _native_fused_numeric_schema_supported(schema) -> bool: + """Return whether every Arrow numeric field is supported by the fused Rust scan.""" + pa = _require_pyarrow() + for field in schema: + dtype = field.type + if pa.types.is_integer(dtype): + if int(dtype.bit_width) not in {8, 16, 32, 64}: + return False + elif pa.types.is_floating(dtype): + if int(dtype.bit_width) not in {32, 64}: + return False + return True + + +def numeric_columns_for_streaming_source(source: StreamingDatasetSource) -> list[str]: + """Return Arrow numeric column names without reading row data.""" + schema_method = getattr(source, "schema", None) + if not callable(schema_method): + raise TypeError("Streaming profile sources must expose an Arrow schema().") + numeric, _, _ = _column_groups(schema_method()) + return numeric + + +def _arrow_numeric_to_float64(array) -> np.ndarray: + """Convert one bounded Arrow numeric array to a contiguous float64 buffer. + + This is a compatibility fallback for NumPy and older native extensions. New + native builds profile supported Arrow primitive arrays directly in Rust. + """ + try: + values = array.to_numpy(zero_copy_only=False) + converted = np.asarray(values, dtype=np.float64) + except (TypeError, ValueError): + series = array.to_pandas() + converted = pd.to_numeric(series, errors="coerce").to_numpy( + dtype="float64", + na_value=np.nan, + ) + return np.ascontiguousarray(converted, dtype=np.float64) + + +def _update_native_string_accumulator(accumulator, array) -> None: + """Feed Arrow UTF-8 buffers to Rust without constructing Python strings.""" + pa = _require_pyarrow() + validity_buffer, offsets_buffer, data_buffer = array.buffers() + validity = ( + np.frombuffer(validity_buffer, dtype=np.uint8) + if validity_buffer is not None + else None + ) + data = ( + np.frombuffer(data_buffer, dtype=np.uint8) + if data_buffer is not None + else np.empty(0, dtype=np.uint8) + ) + + if pa.types.is_large_string(array.type): + offsets = np.frombuffer(offsets_buffer, dtype=np.int64) + accumulator.update_large_utf8( + data, + offsets, + len(array), + validity, + int(array.offset), + ) + else: + offsets = np.frombuffer(offsets_buffer, dtype=np.int32) + accumulator.update_utf8( + data, + offsets, + len(array), + validity, + int(array.offset), + ) + + +def _state_from_payload(payload: dict[str, Any]) -> NumericColumnState: + count = int(payload["count"]) + variance = payload.get("variance") + return NumericColumnState( + count=count, + missing=int(payload["missing"]), + mean=float(payload["mean"]) if count else 0.0, + m2=float( + payload.get( + "m2", + float(variance) * (count - 1) + if variance is not None and count >= 2 + else 0.0, + ) + ), + m3=float(payload.get("m3", 0.0)), + m4=float(payload.get("m4", 0.0)), + minimum=( + float(payload["minimum"]) + if payload.get("minimum") is not None + else None + ), + maximum=( + float(payload["maximum"]) + if payload.get("maximum") is not None + else None + ), + infinite=int(payload["infinite"]), + ) + + +def _round_optional(value: Any, digits: int = 3): + if value is None: + return None + return round(float(value), digits) + + +def _summary_from_native_snapshot(payload: dict[str, Any]) -> dict[str, Any]: + quantiles = payload.get("quantiles", {}) + return { + "count": int(payload["count"]), + "mean": _round_optional(payload.get("mean")), + "std": _round_optional(payload.get("std")), + "min": _round_optional(payload.get("minimum")), + "25%": _round_optional(quantiles.get("p25")), + "50%": _round_optional(quantiles.get("p50")), + "75%": _round_optional(quantiles.get("p75")), + "max": _round_optional(payload.get("maximum")), + "skewness": _round_optional(payload.get("skewness"), 6), + "kurtosis": _round_optional(payload.get("kurtosis"), 6), + } + + +def _summary_from_python_state( + state: NumericColumnState, + sample: pd.Series, + quantile_sketch: NumpyLogQuantileSketch | None = None, +) -> dict[str, Any]: + if quantile_sketch is not None: + q25 = quantile_sketch.quantile(0.25) + q50 = quantile_sketch.quantile(0.50) + q75 = quantile_sketch.quantile(0.75) + else: + finite_sample = pd.to_numeric(sample, errors="coerce") + finite_values = finite_sample.to_numpy(dtype="float64", na_value=np.nan) + finite_sample = finite_sample[np.isfinite(finite_values)] + quantiles = ( + finite_sample.quantile([0.25, 0.5, 0.75]) + if len(finite_sample) + else pd.Series() + ) + q25 = quantiles.get(0.25) + q50 = quantiles.get(0.5) + q75 = quantiles.get(0.75) + return { + "count": state.count, + "mean": _round_optional(state.mean if state.count else None), + "std": _round_optional(state.std), + "min": _round_optional(state.minimum), + "25%": _round_optional(q25), + "50%": _round_optional(q50), + "75%": _round_optional(q75), + "max": _round_optional(state.maximum), + "skewness": _round_optional(state.skewness, 6), + "kurtosis": _round_optional(state.kurtosis, 6), + } + + +def _sample_positions(rows: int, target_rows: int) -> np.ndarray: + return _deterministic_stratified_positions(rows, target_rows) + + +def _sample_batch(batch, positions: np.ndarray, offset: int): + pa = _require_pyarrow() + if positions.size == 0 or batch.num_rows == 0: + return None + left = int(np.searchsorted(positions, offset, side="left")) + right = int(np.searchsorted(positions, offset + batch.num_rows, side="left")) + if right <= left: + return None + local = positions[left:right] - offset + return batch.take(pa.array(local, type=pa.int64())).to_pandas() + + +def sample_streaming_source( + source: StreamingDatasetSource, + *, + sample_rows: int, + batch_size: int = STREAM_BATCH_SIZE, + columns: Sequence[str] | None = None, +) -> pd.DataFrame: + """Return a deterministic stratified-jitter sample without full materialization.""" + if sample_rows < 1: + raise ValueError("sample_rows must be at least 1.") + if batch_size < 1: + raise ValueError("batch_size must be at least 1.") + + metadata = source.inspect() + rows = metadata.rows + if rows is None: + raise ValueError("Streaming sampling requires a source row count.") + if rows < 1: + raise ValueError(f"Dataset is empty: {metadata.name}") + + projected_width = len(columns) if columns is not None else int(metadata.columns or 1) + effective_batch_size = _width_aware_row_limit( + int(batch_size), + projected_width, + cell_budget=STREAM_BATCH_CELL_BUDGET, + ) + effective_sample_rows = _width_aware_row_limit( + int(sample_rows), + projected_width, + cell_budget=STREAM_SAMPLE_CELL_BUDGET, + ) + + positions = _sample_positions(int(rows), effective_sample_rows) + frames: list[pd.DataFrame] = [] + offset = 0 + for batch in source.iter_batches(batch_size=effective_batch_size, columns=columns): + sampled = _sample_batch(batch, positions, offset) + if sampled is not None and not sampled.empty: + frames.append(sampled) + offset += int(batch.num_rows) + + if offset != rows: + raise ValueError( + f"Streaming source metadata reported {rows} rows but yielded {offset}." + ) + + if frames: + sample = pd.concat(frames, ignore_index=True) + return sample.head(len(positions)) + return pd.DataFrame(columns=list(columns or ())) + + +def _duplicate_summary(sample: pd.DataFrame, rows: int) -> tuple[int, dict[str, Any]]: + if rows == len(sample): + count = int(sample.duplicated().sum()) + return count, { + "method": "exact_streaming_sample", + "sampled": False, + "sample_rows": int(len(sample)), + } + + sample_duplicates = int(sample.duplicated().sum()) + rate = sample_duplicates / max(len(sample), 1) + estimate = int(round(rate * rows)) + return estimate, { + "method": "streaming_sample_estimate", + "sampled": True, + "sample_rows": int(len(sample)), + "source_rows": int(rows), + "estimated_duplicate_rate": round(float(rate), 6), + } + + +def _estimated_memory_usage(sample: pd.DataFrame, rows: int) -> tuple[float, dict[str, Any]]: + if len(sample) == 0: + return 0.0, {"method": "unavailable", "estimated": True} + sample_bytes = int(sample.memory_usage(index=True, deep=True).sum()) + if len(sample) == rows: + estimated_bytes = sample_bytes + method = "exact_from_full_stream_sample" + estimated = False + else: + estimated_bytes = int(round(sample_bytes / len(sample) * rows)) + method = "scaled_from_stratified_jitter_sample" + estimated = True + return round(estimated_bytes / (1024 * 1024), 3), { + "method": method, + "estimated": estimated, + "sample_rows": int(len(sample)), + "source_rows": int(rows), + } + + +def build_streaming_profile( + source: StreamingDatasetSource, + *, + batch_size: int = STREAM_BATCH_SIZE, + sample_rows: int = STREAM_SAMPLE_ROWS, + return_sample: bool = False, +) -> dict[str, Any] | tuple[dict[str, Any], pd.DataFrame]: + """Profile a streaming Arrow source without materializing all rows.""" + if batch_size < 1: + raise ValueError("batch_size must be at least 1.") + if sample_rows < 1: + raise ValueError("sample_rows must be at least 1.") + + metadata = source.inspect() + rows = metadata.rows + if rows is None: + raise ValueError("Streaming profiling currently requires a source row count.") + if rows < 1: + raise ValueError(f"Dataset is empty: {metadata.name}") + + schema_method = getattr(source, "schema", None) + if not callable(schema_method): + raise TypeError("Streaming profile sources must expose an Arrow schema().") + schema = schema_method() + columns = [field.name for field in schema] + numeric_cols, categorical_cols, date_cols = _column_groups(schema) + numeric_col_set = set(numeric_cols) + non_numeric_columns = [ + (index, column) + for index, column in enumerate(columns) + if column not in numeric_col_set + ] + string_cols = _string_columns(schema) + dtypes = {field.name: str(field.type) for field in schema} + + width = max(len(columns), 1) + effective_batch_size = _width_aware_row_limit( + int(batch_size), + width, + cell_budget=STREAM_BATCH_CELL_BUDGET, + ) + effective_sample_rows = _width_aware_row_limit( + int(sample_rows), + width, + cell_budget=STREAM_SAMPLE_CELL_BUDGET, + ) + positions = _sample_positions(int(rows), effective_sample_rows) + sample_frames: list[pd.DataFrame] = [] + preview_frame: pd.DataFrame | None = None + missing_counts = {column: 0 for column in columns} + selected_backend = resolve_numeric_backend() + use_numpy_quantile_sketches = ( + selected_backend == "numpy" + and should_use_full_stream_numpy_sketch(int(rows), len(numeric_cols)) + ) + + native_batch_accumulator = None + native_accumulators: dict[str, Any] = {} + native_string_accumulators: dict[str, Any] = {} + python_states = {column: NumericColumnState() for column in numeric_cols} + numpy_quantile_sketches = ( + {column: NumpyLogQuantileSketch() for column in numeric_cols} + if use_numpy_quantile_sketches + else {} + ) + if selected_backend == "rust": + if numeric_cols and _native_fused_numeric_schema_supported(schema): + native_batch_accumulator = create_arrow_batch_profile_accumulator() + if native_batch_accumulator is None: + native_accumulators = { + column: create_numeric_accumulator(stream_id=index) + for index, column in enumerate(numeric_cols) + } + native_string_accumulators = { + column: accumulator + for column in string_cols + if (accumulator := create_string_accumulator()) is not None + } + + native_arrow_fused = native_batch_accumulator is not None + offset = 0 + batches_scanned = 0 + for batch in source.iter_batches(batch_size=effective_batch_size): + batches_scanned += 1 + if preview_frame is None: + preview_frame = batch.slice(0, min(15, batch.num_rows)).to_pandas() + + sampled = _sample_batch(batch, positions, offset) + if sampled is not None and not sampled.empty: + sample_frames.append(sampled) + + if native_arrow_fused: + native_batch_accumulator.update(batch) + for index, column in non_numeric_columns: + array = batch.column(index) + missing_counts[column] += int(array.null_count) + string_accumulator = native_string_accumulators.get(column) + if string_accumulator is not None: + _update_native_string_accumulator(string_accumulator, array) + else: + for index, column in enumerate(columns): + array = batch.column(index) + if column not in numeric_col_set: + missing_counts[column] += int(array.null_count) + string_accumulator = native_string_accumulators.get(column) + if string_accumulator is not None: + _update_native_string_accumulator(string_accumulator, array) + continue + + values = _arrow_numeric_to_float64(array) + accumulator = native_accumulators.get(column) + if accumulator is not None: + accumulator.update_f64(values) + else: + numeric_payload = numeric_state(values, backend="numpy") + python_states[column] = python_states[column].merge( + _state_from_payload(numeric_payload) + ) + quantile_sketch = numpy_quantile_sketches.get(column) + if quantile_sketch is not None: + quantile_sketch.update(values) + + offset += int(batch.num_rows) + + if offset != rows: + raise ValueError( + f"Streaming source metadata reported {rows} rows but yielded {offset}." + ) + + sample = ( + pd.concat(sample_frames, ignore_index=True) + if sample_frames + else pd.DataFrame(columns=columns) + ) + if len(sample) > len(positions): + sample = sample.head(len(positions)) + + numeric_summary: dict[str, dict[str, Any]] = {} + quantile_accuracy = None + if selected_backend == "rust": + if native_arrow_fused: + batch_payload = dict(native_batch_accumulator.snapshot()) + if int(batch_payload.get("rows", -1)) != int(rows): + raise ValueError( + "Native Arrow profiler row count did not match streaming source metadata." + ) + native_profiles = { + str(column): dict(profile) + for column, profile in dict(batch_payload.get("profiles", {})).items() + } + missing_numeric = [ + column for column in numeric_cols if column not in native_profiles + ] + if missing_numeric: + raise RuntimeError( + "Native Arrow profiler skipped supported numeric columns: " + + ", ".join(missing_numeric[:5]) + ) + else: + native_profiles = { + column: dict(native_accumulators[column].snapshot()) + for column in numeric_cols + } + + for column in numeric_cols: + numeric_payload = native_profiles[column] + numeric_summary[column] = _summary_from_native_snapshot(numeric_payload) + missing_counts[column] = int(numeric_payload["missing"]) + quantile_accuracy = numeric_payload.get("quantiles", {}).get( + "relative_accuracy", + quantile_accuracy, + ) + numeric_metadata = { + "backend": "rust", + "method": ( + "native_arrow_fused_record_batch" + if native_arrow_fused + else "native_streaming_accumulator" + ), + "zero_copy_arrow_input": bool(native_arrow_fused), + "approximate_quantiles": True, + "quantile_relative_accuracy": quantile_accuracy, + "quantile_source": "full_stream_sketch", + "finite_only_moments": True, + "higher_moments": "full_stream_exact", + "shape_statistics": ["skewness", "kurtosis"], + "columns_profiled": len(numeric_cols), + "raw_observations_retained": False, + } + else: + for column in numeric_cols: + state = python_states[column] + numeric_summary[column] = _summary_from_python_state( + state, + sample[column] if column in sample else pd.Series(dtype="float64"), + numpy_quantile_sketches.get(column), + ) + missing_counts[column] = state.missing + if use_numpy_quantile_sketches: + numeric_metadata = { + "backend": "numpy", + "method": "mergeable_streaming_moments_with_full_stream_log_quantiles", + "approximate_quantiles": True, + "quantile_relative_accuracy": 0.01, + "quantile_source": "full_stream_sketch", + "quantile_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET), + "finite_only_moments": True, + "higher_moments": "full_stream_exact", + "shape_statistics": ["skewness", "kurtosis"], + "columns_profiled": len(numeric_cols), + "raw_observations_retained": False, + } + else: + numeric_metadata = { + "backend": "numpy", + "method": "mergeable_streaming_moments_with_row_sample_quantiles", + "approximate_quantiles": len(sample) < rows, + "quantile_sample_rows": int(len(sample)), + "quantile_source": "bounded_row_sample", + "quantile_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET), + "quantile_sketch_skipped_for_cost": bool(numeric_cols), + "finite_only_moments": True, + "higher_moments": "full_stream_exact", + "shape_statistics": ["skewness", "kurtosis"], + "columns_profiled": len(numeric_cols), + "raw_observations_retained": False, + } + + categorical_summary: dict[str, dict[str, Any]] = {} + native_categorical_columns: list[str] = [] + sample_categorical_columns: list[str] = [] + for column in categorical_cols: + string_accumulator = native_string_accumulators.get(column) + if string_accumulator is not None: + categorical_payload = dict(string_accumulator.snapshot()) + missing_counts[column] = int(categorical_payload["missing"]) + count = int(categorical_payload["count"]) + estimate = min(int(categorical_payload["cardinality_estimate"]), count) + categorical_summary[column] = { + "unique_values": estimate, + "top_values": { + str(label): int(candidate_count) + for label, candidate_count in categorical_payload["heavy_hitters"][:10] + }, + "approximate": True, + "unique_values_method": categorical_payload["cardinality_method"], + "top_values_method": categorical_payload["heavy_hitter_method"], + "top_values_count_semantics": categorical_payload[ + "heavy_hitter_count_semantics" + ], + } + native_categorical_columns.append(column) + continue + + if column not in sample: + continue + counts = sample[column].value_counts(dropna=False).head(10) + categorical_summary[column] = { + "unique_values": int(sample[column].nunique(dropna=True)), + "top_values": {str(key): int(value) for key, value in counts.items()}, + "approximate": len(sample) < rows, + } + sample_categorical_columns.append(column) + + if native_categorical_columns: + categorical_method = ( + "native_full_stream_sketch" + if not sample_categorical_columns + else "native_full_stream_sketch_with_sample_fallback" + ) + else: + categorical_method = "exact" if len(sample) == rows else "stratified_jitter_row_sample" + categorical_metadata = { + "method": categorical_method, + "sampled": bool(sample_categorical_columns and len(sample) < rows), + "sample_rows": int(len(sample)), + "source_rows": int(rows), + "native_full_stream_columns": native_categorical_columns, + "sample_fallback_columns": sample_categorical_columns, + "cardinality_method": "hyperloglog" if native_categorical_columns else None, + "top_values_count_semantics": ( + "lower_bound_candidates" if native_categorical_columns else None + ), + } + + missing_series = pd.Series(missing_counts, dtype="int64") + missing_percent = (missing_series / max(rows, 1) * 100).round(2) + + duplicate_rows, duplicate_metadata = _duplicate_summary(sample, int(rows)) + correlations, correlation_metadata = _bounded_correlations(sample, numeric_cols) + correlation_metadata.update({ + "row_sampled": len(sample) < rows, + "sample_rows": int(len(sample)), + "source_rows": int(rows), + }) + + memory_usage_mb, memory_usage_metadata = _estimated_memory_usage(sample, int(rows)) + preview_source = preview_frame if preview_frame is not None else pd.DataFrame(columns=columns) + preview_object = preview_source.astype(object) + preview = preview_object.where(preview_object.notna(), None).to_dict(orient="records") + + source_metadata = metadata.to_dict() if hasattr(metadata, "to_dict") else asdict(metadata) + payload = { + "shape": {"rows": int(rows), "columns": len(columns)}, + "columns": columns, + "dtypes": dtypes, + "numeric_columns": numeric_cols, + "categorical_columns": categorical_cols, + "date_columns": date_cols, + "missing_counts": series_to_dict(missing_series), + "missing_percent": series_to_dict(missing_percent), + "duplicate_rows": duplicate_rows, + "duplicate_percent": round(duplicate_rows / max(rows, 1) * 100, 2), + "duplicate_metadata": duplicate_metadata, + "memory_usage_mb": memory_usage_mb, + "memory_usage_metadata": memory_usage_metadata, + "numeric_summary": numeric_summary, + "numeric_summary_metadata": numeric_metadata, + "categorical_summary": categorical_summary, + "categorical_summary_metadata": categorical_metadata, + "correlations": correlations, + "correlation_metadata": correlation_metadata, + "preview": preview, + "source_metadata": source_metadata, + "streaming_metadata": { + "enabled": True, + "full_materialization": False, + "batch_size": int(effective_batch_size), + "requested_batch_size": int(batch_size), + "batch_cell_budget": int(STREAM_BATCH_CELL_BUDGET), + "batches_scanned": int(batches_scanned), + "sample_rows": int(len(sample)), + "sample_row_limit": int(effective_sample_rows), + "requested_sample_rows": int(sample_rows), + "sample_cell_budget": int(STREAM_SAMPLE_CELL_BUDGET), + "sample_cells_retained": int(len(sample) * len(columns)), + "source_columns": int(len(columns)), + "width_limited": bool( + effective_batch_size < int(batch_size) + or effective_sample_rows < int(sample_rows) + ), + "sample_strategy": "stratified_jitter_global_rows", + "numeric_backend": selected_backend, + "native_arrow_fused_numeric": bool(native_arrow_fused), + "numpy_full_stream_quantile_sketches": bool(use_numpy_quantile_sketches), + "numpy_numeric_sketch_cell_budget": int(PYTHON_NUMERIC_SKETCH_CELL_BUDGET), + "native_string_sketch_columns": native_categorical_columns, + }, + } + if return_sample: + return payload, sample + return payload diff --git a/src/framevitals/streaming_quality.py b/src/framevitals/streaming_quality.py new file mode 100644 index 0000000..a1d641e --- /dev/null +++ b/src/framevitals/streaming_quality.py @@ -0,0 +1,211 @@ +"""Quality diagnostics for streaming dataset sources. + +This adapter reuses FrameVitals' deterministic quality checks on a bounded row +sample while preserving full-source profile facts. Findings whose truth cannot +be proven from a sample (for example primary-key uniqueness or duplicate-column +identity) are explicitly reported as candidates rather than full-source facts. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pandas as pd + +from framevitals.column_roles import infer_column_roles +from framevitals.provenance import normalize_execution +from framevitals.quality_diagnostics import run_quality_diagnostics + + +_SAMPLE_FINDING_KEYS = ( + "identifier_duplicates", + "quasi_constant_columns", + "coercion_candidates", + "category_normalisation", + "blank_strings", + "infinite_values", + "mixed_object_types", + "missingness_relationships", +) + + +def _annotate_sample_findings( + payload: dict[str, Any], + *, + source_rows: int, + sample_rows: int, +) -> None: + sampled = sample_rows < source_rows + if not sampled: + return + + for key in _SAMPLE_FINDING_KEYS: + findings = payload.get(key, []) + if not isinstance(findings, list): + continue + for finding in findings: + if not isinstance(finding, dict): + continue + finding["sampled"] = True + finding["sample_rows"] = sample_rows + finding["source_rows"] = source_rows + if key == "identifier_duplicates": + finding["count_semantics"] = "lower_bound_from_sample" + + primary_keys = payload.get("primary_key_candidates", []) + if isinstance(primary_keys, list): + for finding in primary_keys: + if not isinstance(finding, dict): + continue + finding["confidence"] = "candidate" + finding["sampled"] = True + finding["sample_rows"] = sample_rows + finding["source_rows"] = source_rows + finding["full_source_uniqueness_confirmed"] = False + finding["reason"] = ( + "Column is complete and unique within the bounded row sample; " + "full-source uniqueness was not confirmed." + ) + + duplicate_columns = payload.get("duplicate_columns", []) + if isinstance(duplicate_columns, list): + for finding in duplicate_columns: + if not isinstance(finding, dict): + continue + finding["sampled"] = True + finding["sample_rows"] = sample_rows + finding["source_rows"] = source_rows + finding["confirmed_with_full_equality"] = False + finding["candidate_only"] = True + finding["confirmation_scope"] = "bounded_row_sample" + + +def run_streaming_quality_diagnostics( + sample: pd.DataFrame, + *, + profile: Mapping[str, Any], + source_rows: int, + source_columns: int, + max_sample_rows: int = 5_000, + max_columns: int = 100, + max_missingness_columns: int = 25, +) -> dict[str, Any]: + """Run quality diagnostics without materializing a streaming source. + + Full-row profile facts may cover either the complete schema or a projected + ultra-wide schema. Value-level diagnostics operate on ``sample``. Execution + metadata records both row and column coverage so projected checks cannot be + mistaken for full-schema facts. + """ + if source_rows < 1: + raise ValueError("source_rows must be at least 1.") + if source_columns < 1: + raise ValueError("source_columns must be at least 1.") + + roles = infer_column_roles(sample) + payload = run_quality_diagnostics( + sample, + profile=profile, + column_roles=roles, + max_sample_rows=max_sample_rows, + max_columns=max_columns, + max_missingness_columns=max_missingness_columns, + ) + + sample_rows = int(len(sample)) + profiled_columns = int( + profile.get("streaming_metadata", {}).get( + "profiled_columns", + len(profile.get("columns", sample.columns)), + ) + or len(profile.get("columns", sample.columns)) + ) + column_sampled = bool( + profile.get("streaming_metadata", {}).get("column_sampled", False) + ) or profiled_columns < int(source_columns) + + _annotate_sample_findings( + payload, + source_rows=int(source_rows), + sample_rows=sample_rows, + ) + + columns_checked = min(int(len(sample.columns)), int(max_columns)) + payload["rows"] = int(source_rows) + payload["columns"] = int(source_columns) + payload["profiled_columns"] = profiled_columns + payload["columns_checked"] = columns_checked + payload["truncated_columns"] = columns_checked < int(source_columns) + payload["duplicate_rows"] = int(profile.get("duplicate_rows", 0) or 0) + + issue_groups = ( + "identifier_duplicates", + "quasi_constant_columns", + "duplicate_columns", + "coercion_candidates", + "category_normalisation", + "blank_strings", + "infinite_values", + "mixed_object_types", + "missingness_relationships", + ) + issue_count = sum( + len(payload.get(key, [])) + for key in issue_groups + if isinstance(payload.get(key, []), list) + ) + duplicate_rows = int(payload["duplicate_rows"]) + payload["summary"] = { + "issue_groups": sum(bool(payload.get(key)) for key in issue_groups), + "issue_count": issue_count + (1 if duplicate_rows else 0), + "primary_key_candidate_count": len(payload.get("primary_key_candidates", [])), + } + + profile_fact_scope = ( + "full_rows_projected_columns" if column_sampled else "full_stream" + ) + payload["execution"] = normalize_execution( + { + "method": "streaming_profile_with_bounded_quality_sample", + "full_materialization": False, + "source_rows": int(source_rows), + "source_columns": int(source_columns), + "profiled_columns": profiled_columns, + "columns_checked": columns_checked, + "column_sampled": column_sampled, + "sample_rows": sample_rows, + "sampled": sample_rows < int(source_rows), + "profile_fact_scope": profile_fact_scope, + "full_source_inputs": ( + ["missingness", "duplicate_row_estimate"] + if not column_sampled + else [] + ), + "projected_full_row_inputs": ( + ["missingness", "duplicate_row_estimate"] + if column_sampled + else [] + ), + "sample_inputs": [ + "column_roles", + "identifier_duplicates", + "quasi_constants", + "duplicate_column_candidates", + "coercion_candidates", + "category_normalisation", + "blank_strings", + "infinite_values", + "mixed_object_types", + "missingness_relationships", + ], + "candidate_only_checks": ( + ["primary_key_candidates", "duplicate_columns"] + if sample_rows < int(source_rows) or column_sampled + else [] + ), + }, + method="streaming_profile_with_bounded_quality_sample", + full_materialization=False, + ) + return payload diff --git a/src/framevitals/streaming_roles.py b/src/framevitals/streaming_roles.py new file mode 100644 index 0000000..5362613 --- /dev/null +++ b/src/framevitals/streaming_roles.py @@ -0,0 +1,233 @@ +"""Column-role inference for streaming dataset sources. + +Semantic/value-pattern inference remains bounded, while full-source profile facts +correct the sample roles where the streaming profiler has authoritative or +full-stream sketch evidence. Every column records the scope of its cardinality +evidence. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pandas as pd + +from framevitals.column_roles import ( + ID_KEYWORDS, + _classify_missingness, + _name_matches, + infer_column_roles, + summarize_roles, +) +from framevitals.provenance import normalize_execution + + +_CARDINALITY_ROLES = { + "constant", + "binary", + "low_cardinality", + "high_cardinality", + "unique_like", +} +_MISSINGNESS_ROLES = { + "complete", + "low_missing", + "moderate_missing", + "high_missing", + "very_high_missing", + "severe_missing", +} +_DERIVED_ROLES = { + "analysis_candidate", + "target_candidate", + "regression_target_candidate", +} + + +def _categorical_cardinality( + profile: Mapping[str, Any], + column: str, + *, + sampled: bool, +) -> tuple[int | None, str, str, bool]: + summaries = profile.get("categorical_summary", {}) + if not isinstance(summaries, Mapping): + return None, "unavailable", "unavailable", True + raw = summaries.get(column) + if not isinstance(raw, Mapping) or "unique_values" not in raw: + return None, "unavailable", "unavailable", True + + metadata = profile.get("categorical_summary_metadata", {}) + if not isinstance(metadata, Mapping): + metadata = {} + native_columns = set(metadata.get("native_full_stream_columns", []) or []) + fallback_columns = set(metadata.get("sample_fallback_columns", []) or []) + + unique_count = int(raw.get("unique_values", 0) or 0) + approximate = bool(raw.get("approximate", False)) + method = str(raw.get("unique_values_method") or metadata.get("method") or "streaming_profile") + + if column in native_columns: + return unique_count, "full_stream_approximate", method, True + if column in fallback_columns: + scope = "bounded_row_sample" if sampled else "full_source" + fallback_method = "evenly_spaced_row_sample" if sampled else "exact_full_source_sample" + return unique_count, scope, fallback_method, sampled + + # A categorical result without explicit fallback metadata is exact only when + # the retained sample covers the entire source. Otherwise keep it scoped as + # approximate instead of silently promoting a sample statistic. + scope = "bounded_row_sample" if sampled else "full_source" + return unique_count, scope, method, approximate or sampled + + +def _apply_cardinality_roles( + roles: set[str], + *, + column: str, + unique_count: int, + rows: int, + is_numeric: bool, + semantic_type: str | None, +) -> None: + unique_ratio = unique_count / max(rows, 1) + if unique_count <= 1: + roles.add("constant") + if unique_count == 2: + roles.add("binary") + if 2 <= unique_count <= 10: + roles.add("low_cardinality") + if unique_ratio > 0.80: + roles.add("high_cardinality") + if unique_ratio > 0.95: + roles.add("unique_like") + + id_from_name = _name_matches(column, ID_KEYWORDS) + if semantic_type == "uuid": + roles.add("id_like") + elif id_from_name: + if not is_numeric or unique_ratio > 0.99: + roles.add("id_like") + elif unique_ratio > 0.95 and not is_numeric: + roles.add("id_like") + + +def _recompute_derived_roles( + roles: set[str], + *, + is_numeric: bool, + unique_count: int, +) -> None: + roles.difference_update(_DERIVED_ROLES) + excluded = {"id_like", "time_like", "sequence_like", "constant"} + if not roles.intersection(excluded): + roles.add("analysis_candidate") + + if "id_like" not in roles and "constant" not in roles: + if "binary" in roles or "low_cardinality" in roles: + roles.add("target_candidate") + elif is_numeric and unique_count > 10: + roles.add("regression_target_candidate") + + +def infer_streaming_column_roles( + sample: pd.DataFrame, + *, + profile: Mapping[str, Any], +) -> dict[str, Any]: + """Infer roles from a bounded sample corrected by full-stream profile facts.""" + source_rows = int(profile.get("shape", {}).get("rows", len(sample)) or len(sample)) + sample_rows = int(len(sample)) + sampled = sample_rows < source_rows + missing_counts = profile.get("missing_counts", {}) + if not isinstance(missing_counts, Mapping): + missing_counts = {} + + sample_roles = infer_column_roles(sample) + columns: dict[str, Any] = {} + + for column, raw_info in sample_roles.items(): + info = dict(raw_info) + roles = set(info.get("roles", [])) + roles.difference_update(_MISSINGNESS_ROLES) + roles.difference_update(_CARDINALITY_ROLES) + + missing_count = int(missing_counts.get(column, sample[column].isna().sum()) or 0) + non_missing = max(source_rows - missing_count, 0) + missing_percent = round(missing_count / max(source_rows, 1) * 100, 2) + roles.add(_classify_missingness(missing_percent)) + + ( + categorical_unique, + categorical_scope, + categorical_method, + categorical_approximate, + ) = _categorical_cardinality(profile, column, sampled=sampled) + if categorical_unique is not None: + unique_count = categorical_unique + cardinality_scope = categorical_scope + cardinality_method = categorical_method + cardinality_approximate = categorical_approximate + else: + unique_count = int(sample[column].nunique(dropna=True)) + cardinality_scope = "bounded_row_sample" if sampled else "full_source" + cardinality_method = ( + "evenly_spaced_row_sample" if sampled else "exact_full_source_sample" + ) + cardinality_approximate = sampled + + # Sample uniqueness may have introduced id_like. Remove that role unless + # it is justified again by source-corrected cardinality/name/semantics. + roles.discard("id_like") + _apply_cardinality_roles( + roles, + column=str(column), + unique_count=unique_count, + rows=source_rows, + is_numeric=bool(info.get("is_numeric")), + semantic_type=info.get("semantic_type"), + ) + _recompute_derived_roles( + roles, + is_numeric=bool(info.get("is_numeric")), + unique_count=unique_count, + ) + + info.update({ + "roles": sorted(roles), + "missing_percent": missing_percent, + "non_missing_count": non_missing, + "unique_count": unique_count, + "unique_ratio": round(unique_count / max(source_rows, 1), 4), + "cardinality_scope": cardinality_scope, + "cardinality_method": cardinality_method, + "cardinality_approximate": cardinality_approximate, + "semantic_scope": "bounded_row_sample" if sampled else "full_source", + "sample_rows": sample_rows, + "source_rows": source_rows, + }) + columns[str(column)] = info + + execution = normalize_execution( + { + "method": "streaming_profile_with_bounded_semantic_sample", + "full_materialization": False, + "source_rows": source_rows, + "sample_rows": sample_rows, + "sampled": sampled, + "full_source_inputs": ["missingness", "native_categorical_sketches"], + "sample_inputs": [ + "semantic_patterns", + "numeric_cardinality", + "categorical_cardinality_without_native_sketch", + ], + }, + method="streaming_profile_with_bounded_semantic_sample", + full_materialization=False, + ) + return { + "columns": columns, + "summary": summarize_roles(columns), + "execution": execution, + } diff --git a/src/framevitals/streaming_sketches.py b/src/framevitals/streaming_sketches.py new file mode 100644 index 0000000..36f8857 --- /dev/null +++ b/src/framevitals/streaming_sketches.py @@ -0,0 +1,155 @@ +"""Pure-NumPy bounded-memory sketches for non-native streaming execution. + +The Rust backend already owns the fastest full-stream sketch implementation. +These fallbacks mirror the same logarithmic-quantile semantics for datasets +small enough that Python/NumPy sketch maintenance is cheaper than retaining a +large row sample. Ultra-wide inputs deliberately keep the existing sampled +fallback so this module never turns a memory fix into a CPU regression. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np + + +PYTHON_NUMERIC_SKETCH_CELL_BUDGET = 50_000_000 +DEFAULT_RELATIVE_ACCURACY = 0.01 +DEFAULT_ZERO_THRESHOLD = 1.0e-12 + + +def should_use_full_stream_numpy_sketch( + rows: int, + numeric_columns: int, + *, + cell_budget: int = PYTHON_NUMERIC_SKETCH_CELL_BUDGET, +) -> bool: + """Return whether NumPy sketch work fits the configured cell-cost budget.""" + if rows < 0 or numeric_columns < 0: + raise ValueError("rows and numeric_columns must be non-negative.") + if cell_budget < 1: + raise ValueError("cell_budget must be positive.") + return int(rows) * int(numeric_columns) <= int(cell_budget) + + +@dataclass(slots=True) +class NumpyLogQuantileSketch: + """Mergeable relative-accuracy logarithmic quantile sketch. + + This mirrors FrameVitals' native ``LogQuantileSketch`` rather than retaining + raw observations. Batch updates are vectorized with NumPy and only unique + logarithmic bins are folded into Python dictionaries. + """ + + relative_accuracy: float = DEFAULT_RELATIVE_ACCURACY + zero_threshold: float = DEFAULT_ZERO_THRESHOLD + negative: dict[int, int] = field(default_factory=dict) + positive: dict[int, int] = field(default_factory=dict) + zero_count: int = 0 + count: int = 0 + + def __post_init__(self) -> None: + if not 0.0 < float(self.relative_accuracy) < 1.0: + raise ValueError("relative_accuracy must be between 0 and 1.") + if self.zero_threshold < 0: + raise ValueError("zero_threshold must be non-negative.") + + @property + def _log_gamma(self) -> float: + gamma = (1.0 + self.relative_accuracy) / (1.0 - self.relative_accuracy) + return float(np.log(gamma)) + + def _update_side(self, target: dict[int, int], magnitudes: np.ndarray) -> None: + if magnitudes.size == 0: + return + keys = np.floor(np.log(magnitudes) / self._log_gamma).astype(np.int64) + unique, counts = np.unique(keys, return_counts=True) + for key, amount in zip(unique.tolist(), counts.tolist(), strict=True): + integer_key = int(key) + target[integer_key] = target.get(integer_key, 0) + int(amount) + + def update(self, values: Any) -> "NumpyLogQuantileSketch": + array = np.asarray(values, dtype=np.float64).reshape(-1) + finite = array[np.isfinite(array)] + if finite.size == 0: + return self + + self.count += int(finite.size) + absolute = np.abs(finite) + zero_mask = absolute <= self.zero_threshold + self.zero_count += int(np.count_nonzero(zero_mask)) + + nonzero = finite[~zero_mask] + if nonzero.size: + negative = nonzero[nonzero < 0.0] + positive = nonzero[nonzero > 0.0] + self._update_side(self.negative, -negative) + self._update_side(self.positive, positive) + return self + + def merge(self, other: "NumpyLogQuantileSketch") -> "NumpyLogQuantileSketch": + if not np.isclose(self.relative_accuracy, other.relative_accuracy): + raise ValueError("quantile sketch accuracy mismatch") + if not np.isclose(self.zero_threshold, other.zero_threshold): + raise ValueError("quantile sketch zero-threshold mismatch") + self.count += other.count + self.zero_count += other.zero_count + for key, amount in other.negative.items(): + self.negative[key] = self.negative.get(key, 0) + amount + for key, amount in other.positive.items(): + self.positive[key] = self.positive.get(key, 0) + amount + return self + + def _representative(self, key: int) -> float: + return float(np.exp((float(key) + 0.5) * self._log_gamma)) + + def quantile(self, q: float) -> float | None: + if self.count == 0: + return None + if not 0.0 <= float(q) <= 1.0: + raise ValueError("q must be between 0 and 1.") + + target = int(np.floor(float(q) * (self.count - 1))) + seen = 0 + for key in sorted(self.negative, reverse=True): + amount = self.negative[key] + if target < seen + amount: + return -self._representative(key) + seen += amount + + if target < seen + self.zero_count: + return 0.0 + seen += self.zero_count + + for key in sorted(self.positive): + amount = self.positive[key] + if target < seen + amount: + return self._representative(key) + seen += amount + + if self.positive: + return self._representative(max(self.positive)) + if self.negative: + return -self._representative(min(self.negative)) + return 0.0 + + @property + def bin_count(self) -> int: + return len(self.negative) + len(self.positive) + int(self.zero_count > 0) + + def snapshot(self) -> dict[str, Any]: + return { + "method": "numpy_log_quantile_sketch", + "count": int(self.count), + "relative_accuracy": float(self.relative_accuracy), + "bin_count": int(self.bin_count), + "p01": self.quantile(0.01), + "p05": self.quantile(0.05), + "p25": self.quantile(0.25), + "p50": self.quantile(0.50), + "p75": self.quantile(0.75), + "p95": self.quantile(0.95), + "p99": self.quantile(0.99), + } diff --git a/src/framevitals/streaming_target.py b/src/framevitals/streaming_target.py new file mode 100644 index 0000000..5b2834b --- /dev/null +++ b/src/framevitals/streaming_target.py @@ -0,0 +1,101 @@ +"""Target-aware diagnostics for streaming dataset sources.""" + +from __future__ import annotations + +from typing import Any + +from framevitals.column_roles import infer_column_roles +from framevitals.execution import derive_execution_budget +from framevitals.sources import StreamingDatasetSource +from framevitals.streaming_profile import sample_streaming_source +from framevitals.target_intelligence import run_target_intelligence + + +def _column_names(source: StreamingDatasetSource) -> list[str]: + schema_method = getattr(source, "schema", None) + if callable(schema_method): + schema = schema_method() + names = getattr(schema, "names", None) + if names is not None: + return [str(name) for name in names] + + first_batch = next(source.iter_batches(batch_size=1), None) + if first_batch is None: + return [] + schema = getattr(first_batch, "schema", None) + names = getattr(schema, "names", None) + if names is None: + return [] + return [str(name) for name in names] + + +def run_streaming_target_analysis( + source: StreamingDatasetSource, + *, + target: str, + max_columns: int = 200, +) -> dict[str, Any]: + """Run target diagnostics on a deterministic bounded source projection. + + The target is always retained. Feature columns are kept in source order up + to ``max_columns - 1``. This prevents an ultra-wide source from turning a + target-only diagnostic request into an unbounded pandas materialization. + """ + if max_columns < 2: + raise ValueError("max_columns must be at least 2.") + + metadata = source.inspect() + if metadata.rows is None or metadata.columns is None: + raise ValueError("Streaming target analysis requires source shape metadata.") + source_rows = int(metadata.rows) + source_columns = int(metadata.columns) + + names = _column_names(source) + if target not in names: + raise ValueError(f"Target column not found: {target}") + + features = [name for name in names if name != target] + selected_features = features[: max_columns - 1] + selected_set = set(selected_features) + selected_columns = [ + name for name in names if name == target or name in selected_set + ] + + budget = derive_execution_budget( + source_rows, + source_columns, + mode="standard", + ) + sample_limit = max(100, int(budget.pair_sample_rows)) + sample = sample_streaming_source( + source, + sample_rows=sample_limit, + columns=selected_columns, + ) + + column_roles = infer_column_roles(sample) + payload = run_target_intelligence( + sample, + target_column=target, + column_roles=column_roles, + ) + payload["execution"] = { + "scope": ( + "bounded_row_and_column_sample" + if len(sample) < source_rows or len(selected_columns) < source_columns + else "full_source" + ), + "full_materialization": False, + "source_rows": source_rows, + "source_columns": source_columns, + "sample_rows": int(len(sample)), + "sampled_rows": bool(len(sample) < source_rows), + "selected_columns": int(len(selected_columns)), + "feature_columns_considered": int(len(selected_features)), + "feature_columns_available": int(len(features)), + "columns_truncated": bool(len(selected_columns) < source_columns), + "strategy": "streaming_evenly_spaced_rows_with_deterministic_feature_projection", + "target_always_retained": True, + } + payload["source"] = metadata.to_dict() + return payload diff --git a/src/framevitals/target_intelligence.py b/src/framevitals/target_intelligence.py new file mode 100644 index 0000000..9cca085 --- /dev/null +++ b/src/framevitals/target_intelligence.py @@ -0,0 +1,334 @@ +"""Unified target-aware diagnostics for supervised-learning datasets. + +This layer connects FrameVitals' existing target and leakage diagnostics and +adds lightweight, interpretable feature-to-target association ranking. It is +not an AutoML trainer and does not fit production models. +""" + +from __future__ import annotations + +import math +from typing import Any + +import pandas as pd +from scipy import stats + +from framevitals.column_roles import infer_column_roles +from framevitals.target_analyzer import analyze_target +from framevitals.target_leakage import run_target_leakage_analysis + + +_EXCLUDED_ROLES = {"id_like", "constant"} + + +def _safe_float(value: Any) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(number): + return None + return round(number, 6) + + +def _cramers_v(feature: pd.Series, target: pd.Series) -> tuple[float | None, int]: + mask = feature.notna() & target.notna() + n = int(mask.sum()) + if n < 10: + return None, n + + table = pd.crosstab(feature[mask], target[mask]) + if table.shape[0] < 2 or table.shape[1] < 2: + return None, n + if table.shape[0] > 50 or table.shape[1] > 50: + return None, n + + try: + chi2, _, _, _ = stats.chi2_contingency(table, correction=False) + except ValueError: + return None, n + + phi2 = chi2 / n + rows, cols = table.shape + denominator = min(rows - 1, cols - 1) + if denominator <= 0: + return None, n + return _safe_float(math.sqrt(max(phi2, 0.0) / denominator)), n + + +def _correlation_ratio(categories: pd.Series, values: pd.Series) -> tuple[float | None, int]: + mask = categories.notna() & values.notna() + n = int(mask.sum()) + if n < 10: + return None, n + + groups = categories[mask] + numeric = pd.to_numeric(values[mask], errors="coerce") + valid = numeric.notna() + groups = groups[valid] + numeric = numeric[valid] + n = len(numeric) + if n < 10 or groups.nunique(dropna=True) < 2 or groups.nunique(dropna=True) > 50: + return None, n + + grand_mean = float(numeric.mean()) + total = float(((numeric - grand_mean) ** 2).sum()) + if total <= 0: + return None, n + + between = 0.0 + for _, group_values in numeric.groupby(groups): + if group_values.empty: + continue + between += len(group_values) * (float(group_values.mean()) - grand_mean) ** 2 + + eta_squared = max(0.0, min(1.0, between / total)) + return _safe_float(math.sqrt(eta_squared)), n + + +def _numeric_association(feature: pd.Series, target: pd.Series) -> tuple[float | None, int, str]: + mask = feature.notna() & target.notna() + n = int(mask.sum()) + if n < 10: + return None, n, "spearman" + + x = pd.to_numeric(feature[mask], errors="coerce") + y = pd.to_numeric(target[mask], errors="coerce") + valid = x.notna() & y.notna() + x = x[valid] + y = y[valid] + n = len(x) + if n < 10 or x.nunique() < 2 or y.nunique() < 2: + return None, n, "spearman" + + try: + coefficient, _ = stats.spearmanr(x, y) + except Exception: + return None, n, "spearman" + return _safe_float(abs(coefficient)), n, "spearman" + + +def _binary_numeric_association( + feature: pd.Series, + target: pd.Series, +) -> tuple[float | None, int, str]: + mask = feature.notna() & target.notna() + n = int(mask.sum()) + if n < 10: + return None, n, "point_biserial" + + classes = list(pd.unique(target[mask])) + if len(classes) != 2: + score, n_eta = _correlation_ratio(target, feature) + return score, n_eta, "correlation_ratio" + + encoded = target[mask].map({classes[0]: 0, classes[1]: 1}) + numeric = pd.to_numeric(feature[mask], errors="coerce") + valid = numeric.notna() & encoded.notna() + numeric = numeric[valid] + encoded = encoded[valid] + n = len(numeric) + if n < 10 or numeric.nunique() < 2: + return None, n, "point_biserial" + + try: + coefficient, _ = stats.pointbiserialr(encoded.astype(float), numeric.astype(float)) + except Exception: + return None, n, "point_biserial" + return _safe_float(abs(coefficient)), n, "point_biserial" + + +def _association_strength(score: float | None) -> str: + if score is None: + return "unknown" + if score >= 0.8: + return "very_strong" + if score >= 0.6: + return "strong" + if score >= 0.4: + return "moderate" + if score >= 0.2: + return "weak" + return "very_weak" + + +def _rank_target_associations( + df: pd.DataFrame, + *, + target_column: str, + task_type: str, + column_roles: dict, + max_features: int, +) -> list[dict[str, Any]]: + target = df[target_column] + target_is_numeric = pd.api.types.is_numeric_dtype(target) + associations: list[dict[str, Any]] = [] + + for column in df.columns: + if column == target_column: + continue + + role_info = column_roles.get(column, {}) + role_set = set(role_info.get("roles", [])) + if role_set.intersection(_EXCLUDED_ROLES): + continue + + feature = df[column] + feature_is_numeric = pd.api.types.is_numeric_dtype(feature) + score: float | None = None + overlap = 0 + method = "unsupported" + + if task_type == "regression" and target_is_numeric: + if feature_is_numeric: + score, overlap, method = _numeric_association(feature, target) + else: + score, overlap = _correlation_ratio(feature, target) + method = "correlation_ratio" + elif task_type == "classification": + if feature_is_numeric: + score, overlap, method = _binary_numeric_association(feature, target) + else: + score, overlap = _cramers_v(feature, target) + method = "cramers_v" + + if score is None: + continue + + associations.append({ + "feature": column, + "score": score, + "strength": _association_strength(score), + "method": method, + "overlap": int(overlap), + }) + + associations.sort(key=lambda item: (-item["score"], item["feature"])) + return associations[:max_features] + + +def _split_guidance( + target_profile: dict[str, Any], + column_roles: dict, +) -> dict[str, Any]: + time_like = [ + column + for column, info in column_roles.items() + if "time_like" in info.get("roles", []) + ] + task_type = target_profile.get("task_type") + details = target_profile.get("details", {}) or {} + + if time_like: + return { + "strategy": "review_time_aware_split", + "reason": ( + "Time-like columns are present. If rows represent chronological " + "events, prefer an ordered/time-based split to avoid future-to-past leakage." + ), + "time_candidates": time_like[:10], + } + + if task_type == "classification" and details.get("class_count", 0) >= 2: + return { + "strategy": "stratified_random_split", + "reason": "Preserve target-class proportions across train/validation/test splits.", + "time_candidates": [], + } + + if task_type == "regression": + return { + "strategy": "random_split", + "reason": ( + "No time-like structure was detected; a reproducible random split is a " + "reasonable baseline for regression." + ), + "time_candidates": [], + } + + return { + "strategy": "review_manually", + "reason": "FrameVitals could not infer a safe split strategy from the target metadata.", + "time_candidates": [], + } + + +def run_target_intelligence( + df: pd.DataFrame, + *, + target_column: str | None, + column_roles: dict | None = None, + max_features: int = 25, +) -> dict[str, Any]: + """Run explainable target-quality, leakage, association, and split checks.""" + if not target_column or target_column not in df.columns: + return { + "available": False, + "message": "No valid target column selected.", + } + if max_features < 1: + raise ValueError("max_features must be at least 1.") + + if column_roles is None: + column_roles = infer_column_roles(df) + + target_profile = analyze_target(df, target_column) + leakage = run_target_leakage_analysis(df, target_column) + task_type = target_profile.get("task_type", "unknown") + associations = _rank_target_associations( + df, + target_column=target_column, + task_type=task_type, + column_roles=column_roles, + max_features=max_features, + ) + + target_roles = column_roles.get(target_column, {}).get("roles", []) + warnings: list[dict[str, str]] = [] + + if "id_like" in target_roles: + warnings.append({ + "code": "target.id_like", + "severity": "high", + "message": "Selected target looks identifier-like and may not be a meaningful prediction target.", + }) + if target_profile.get("missing_percent", 0) >= 20: + warnings.append({ + "code": "target.high_missingness", + "severity": "high", + "message": f"Target has {target_profile['missing_percent']}% missing values.", + }) + + details = target_profile.get("details", {}) or {} + if task_type == "classification" and details.get("class_count", 0) > 50: + warnings.append({ + "code": "target.high_cardinality_classification", + "severity": "medium", + "message": ( + f"Target has {details['class_count']} classes; confirm this is intentional " + "before treating the task as classification." + ), + }) + + for item in leakage.get("warnings", []): + warnings.append({ + "code": f"target.leakage.{item.get('feature', 'feature')}", + "severity": str(item.get("risk", "medium")).lower(), + "message": str(item.get("reason", "Potential target leakage detected.")), + }) + + severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3} + warnings.sort(key=lambda item: (severity_order.get(item["severity"], 9), item["code"])) + + return { + "available": True, + "target_column": target_column, + "task_type": task_type, + "target_profile": target_profile, + "target_roles": list(target_roles), + "split_guidance": _split_guidance(target_profile, column_roles), + "leakage": leakage, + "top_associations": associations, + "warning_count": len(warnings), + "warnings": warnings, + } diff --git a/src/framevitals/target_leakage.py b/src/framevitals/target_leakage.py index a4f9829..ead8e83 100644 --- a/src/framevitals/target_leakage.py +++ b/src/framevitals/target_leakage.py @@ -32,13 +32,66 @@ def numeric_correlation(a, b): return None, overlap -def classify_target_leakage_risk(feature, target, same_ratio, corr): +def categorical_mapping_purity(feature, target, *, max_categories=20): + """Measure whether low-cardinality feature values determine target labels. + + A value of 1.0 means that every observed feature category maps to exactly one + target class. High-cardinality columns are intentionally skipped because + identifiers can trivially memorize a target without representing a useful + categorical leakage pattern. + """ + both_present = feature.notna() & target.notna() + overlap = int(both_present.sum()) + if overlap < 10: + return None, overlap + + feature_values = feature.loc[both_present] + target_values = target.loc[both_present] + feature_cardinality = int(feature_values.nunique(dropna=True)) + target_cardinality = int(target_values.nunique(dropna=True)) + + if ( + feature_cardinality < 2 + or feature_cardinality > max_categories + or target_cardinality < 2 + or target_cardinality > max_categories + ): + return None, overlap + + table = pd.crosstab(feature_values, target_values) + if table.empty: + return None, overlap + + correctly_determined = int(table.max(axis=1).sum()) + purity = correctly_determined / max(int(table.to_numpy().sum()), 1) + return round(float(purity), 4), overlap + + +def classify_target_leakage_risk( + feature, + target, + same_ratio, + corr, + mapping_purity=None, +): lower_feature = feature.lower() lower_target = target.lower() if same_ratio is not None and same_ratio >= 0.98: return "Critical", "Feature values almost exactly match the target on non-missing rows." + if mapping_purity is not None and mapping_purity >= 0.995: + return ( + "Critical", + "Feature categories almost perfectly determine the target labels.", + ) + + if mapping_purity is not None and mapping_purity >= 0.98: + return ( + "High", + "Feature categories very strongly determine the target labels.", + ) + if corr is not None and abs(corr) >= 0.98: return "High", "Feature is almost perfectly correlated with the target." @@ -73,11 +126,17 @@ def run_target_leakage_analysis(df, target_column): if pd.api.types.is_numeric_dtype(feature) and pd.api.types.is_numeric_dtype(target): corr, corr_overlap = numeric_correlation(feature, target) + mapping_purity = None + mapping_overlap = 0 + if not pd.api.types.is_numeric_dtype(feature): + mapping_purity, mapping_overlap = categorical_mapping_purity(feature, target) + risk, reason = classify_target_leakage_risk( feature=column, target=target_column, same_ratio=same_ratio, corr=corr, + mapping_purity=mapping_purity, ) if risk in {"Critical", "High", "Medium"}: @@ -91,6 +150,8 @@ def run_target_leakage_analysis(df, target_column): "same_overlap": same_overlap, "correlation": corr, "correlation_overlap": corr_overlap, + "mapping_purity": mapping_purity, + "mapping_overlap": mapping_overlap, } ) diff --git a/src/framevitals/time_series.py b/src/framevitals/time_series.py index 5b88311..8de00a8 100644 --- a/src/framevitals/time_series.py +++ b/src/framevitals/time_series.py @@ -243,7 +243,6 @@ def _guess_period_via_fft(values: np.ndarray, freq_label: str) -> int | None: # --------------------------------------------------------------------------- def _stationarity(series: pd.Series) -> dict: - out: dict[str, Any] = {"available": False} s = series.dropna() if len(s) < 30: return {"available": False, "reason": "n<30"} diff --git a/src/framevitals/visualizer.py b/src/framevitals/visualizer.py index 9d23597..f6c1fbb 100644 --- a/src/framevitals/visualizer.py +++ b/src/framevitals/visualizer.py @@ -12,7 +12,7 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt -from matplotlib.patches import Wedge, Patch +from matplotlib.patches import Patch import seaborn as sns import numpy as np import pandas as pd @@ -484,7 +484,7 @@ def _chart_pareto_categorical(did, df, col): cum = counts.cumsum() / counts.sum() * 100 fig, ax1 = plt.subplots(figsize=(10, 5.2)) - bars = ax1.bar(range(len(counts)), counts.values, color=ACCENT, + ax1.bar(range(len(counts)), counts.values, color=ACCENT, edgecolor=BG_PAGE, linewidth=1.0, width=0.65, zorder=3) ax1.set_xticks(range(len(counts))) ax1.set_xticklabels([str(x)[:18] for x in counts.index], @@ -1020,7 +1020,7 @@ def generate_charts( chart = _chart_time_series_trend(dataset_id, df, time_series) elif t == "bivariate_highlights": chart = _chart_bivariate_highlights(dataset_id, deep_statistics_v2) - except Exception as exc: # noqa: BLE001 — never let a single chart kill the run + except Exception: # noqa: BLE001 — never let a single chart kill the run chart = None # We deliberately don't log here at WARN level because chart fallout # is expected (e.g. all-NaN columns, single-class targets, etc.). diff --git a/tests/test_acceleration.py b/tests/test_acceleration.py new file mode 100644 index 0000000..fe8bbeb --- /dev/null +++ b/tests/test_acceleration.py @@ -0,0 +1,68 @@ +import framevitals as fv +import framevitals.acceleration as acceleration + + +def test_cupy_recommendation_is_platform_and_cuda_aware(): + assert acceleration._recommend_cupy_package( + "Linux", "13.0", has_nvidia=True + ) == "cupy-cuda13x[ctk]" + assert acceleration._recommend_cupy_package( + "Windows", "12.8", has_nvidia=True + ) == "cupy-cuda12x[ctk]" + assert acceleration._recommend_cupy_package( + "Darwin", "13.0", has_nvidia=True + ) is None + assert acceleration._recommend_cupy_package( + "Linux", None, has_nvidia=True + ) is None + assert acceleration._recommend_cupy_package( + "Linux", "13.0", has_nvidia=False + ) is None + + +def test_nvidia_smi_parser_is_best_effort(monkeypatch): + def fake_smi(arguments): + if arguments: + return "NVIDIA Test GPU, 24576, 999.1" + return "NVIDIA-SMI ... CUDA Version: 13.0" + + monkeypatch.setattr(acceleration, "_run_nvidia_smi", fake_smi) + + devices, cuda = acceleration._detect_nvidia() + + assert cuda == "13.0" + assert len(devices) == 1 + assert devices[0].name == "NVIDIA Test GPU" + assert devices[0].memory_total_mb == 24576 + assert devices[0].driver_version == "999.1" + + +def test_system_info_probe_can_run_without_gpu_initialization(): + result = fv.system_info(probe_gpu=False) + + assert result["platform"] + assert result["architecture"] + assert result["default_cpu_backend"] in {"numpy", "rust"} + assert result["gpu_acceleration"]["automatic_install_performed"] is False + assert isinstance(result["eligible_backends"], list) + + +def test_gpu_install_is_only_recommended_not_performed(monkeypatch): + monkeypatch.setattr(acceleration.platform, "system", lambda: "Linux") + monkeypatch.setattr( + acceleration, + "_detect_nvidia", + lambda: ((acceleration.GpuDevice("GPU", 16384, "1"),), "13.0"), + ) + monkeypatch.setattr( + acceleration, + "_probe_cupy", + lambda: (False, False, None), + ) + + result = acceleration.system_info(probe_gpu=True) + + assert result["gpu_acceleration"]["available"] is False + assert result["gpu_acceleration"]["installable"] is True + assert result["gpu_acceleration"]["recommended_package"] == "cupy-cuda13x[ctk]" + assert result["gpu_acceleration"]["automatic_install_performed"] is False diff --git a/tests/test_adaptive_execution.py b/tests/test_adaptive_execution.py new file mode 100644 index 0000000..31d01a8 --- /dev/null +++ b/tests/test_adaptive_execution.py @@ -0,0 +1,252 @@ +import numpy as np +import pandas as pd +import pytest + +from framevitals.analysis_state import AnalysisState, NumericColumnState +from framevitals.budgeted_analysis import ( + run_budgeted_anomalies, + run_budgeted_deep_statistics, + run_budgeted_time_series, +) +from framevitals.execution import derive_execution_budget, deterministic_sample_frame + + +def test_large_budget_limits_memory_heavy_parallelism(): + budget = derive_execution_budget(100_000, 105, mode="standard") + + assert budget.scale_class == "large" + assert budget.large_dataset is True + assert budget.max_memory_heavy_parallelism == 1 + assert budget.bootstrap_sample_rows < budget.rows + assert budget.time_series_sample_rows < budget.rows + + +def test_extreme_shape_is_classified_without_allocating_dataset(): + budget = derive_execution_budget(100_000_000, 100_000, mode="standard") + + assert budget.scale_class == "extreme" + assert budget.ultra_wide_dataset is True + assert budget.cells == 10_000_000_000_000 + assert budget.max_memory_heavy_parallelism == 1 + + +def test_deterministic_sample_is_bounded_and_covers_endpoints(): + frame = pd.DataFrame({"x": np.arange(10_000)}) + sampled, metadata = deterministic_sample_frame(frame, 1_000) + + assert len(sampled) == 1_000 + assert sampled.iloc[0]["x"] == 0 + assert sampled.iloc[-1]["x"] == 9_999 + assert metadata["sampled"] is True + assert metadata["source_rows"] == 10_000 + assert metadata["sample_rows"] == 1_000 + + +def test_numeric_state_merge_matches_full_frame_statistics(): + frame = pd.DataFrame({ + "x": [1.0, 2.0, np.nan, 4.0, np.inf, 7.0, 9.0], + "y": [10, 11, 12, 13, 14, 15, 16], + }) + + full = AnalysisState.from_frame(frame) + left = AnalysisState.from_frame(frame.iloc[:3]) + right = AnalysisState.from_frame(frame.iloc[3:]) + merged = left.merge(right) + + assert merged.rows == full.rows + assert merged.schema == full.schema + for name in ("x", "y"): + actual = merged.numeric[name] + expected = full.numeric[name] + assert actual.count == expected.count + assert actual.missing == expected.missing + assert actual.infinite == expected.infinite + assert actual.mean == pytest.approx(expected.mean) + assert actual.variance == pytest.approx(expected.variance) + assert actual.minimum == expected.minimum + assert actual.maximum == expected.maximum + + +def test_numeric_state_rejects_incompatible_schema_merge(): + left = AnalysisState.from_frame(pd.DataFrame({"x": [1, 2]})) + right = AnalysisState.from_frame(pd.DataFrame({"x": [1.0, 2.0]})) + + with pytest.raises(ValueError, match="different schemas"): + left.merge(right) + + +def test_deep_statistics_adapter_never_passes_unbounded_frame(monkeypatch): + seen = {} + + def fake_deep(frame, max_pairs=20): + seen["rows"] = len(frame) + seen["pairs"] = max_pairs + return {"available": True} + + monkeypatch.setattr( + "framevitals.budgeted_analysis.run_fast_deep_statistics_v2", + fake_deep, + ) + frame = pd.DataFrame({"x": np.arange(20_000), "y": np.arange(20_000)}) + budget = derive_execution_budget(len(frame), len(frame.columns), mode="standard") + + result = run_budgeted_deep_statistics(frame, budget=budget) + + assert seen["rows"] <= budget.bootstrap_sample_rows + assert seen["pairs"] <= budget.relationship_pair_budget + assert result["execution"]["sampled"] is True + assert result["execution"]["source_rows"] == 20_000 + assert ( + result["execution"]["inference_strategy"] + == "closed_form_and_order_statistics" + ) + + +def test_research_deep_statistics_adapter_keeps_bca_for_small_samples(monkeypatch): + seen = {"bca": 0, "fast": 0} + + def fake_bca(frame, max_pairs=20): + seen["bca"] += 1 + seen["rows"] = len(frame) + seen["pairs"] = max_pairs + return {"available": True} + + def fake_fast(frame, max_pairs=20): + seen["fast"] += 1 + return {"available": True} + + monkeypatch.setattr( + "framevitals.budgeted_analysis.run_deep_statistics_v2", + fake_bca, + ) + monkeypatch.setattr( + "framevitals.budgeted_analysis.run_fast_deep_statistics_v2", + fake_fast, + ) + frame = pd.DataFrame({"x": np.arange(500), "y": np.arange(500)}) + budget = derive_execution_budget(len(frame), len(frame.columns), mode="research") + + result = run_budgeted_deep_statistics(frame, budget=budget) + + assert seen["bca"] == 1 + assert seen["fast"] == 0 + assert seen["rows"] == len(frame) + assert seen["pairs"] <= budget.relationship_pair_budget + assert result["execution"]["inference_strategy"] == "bca_bootstrap_small_sample" + + +def test_research_deep_statistics_adapter_uses_fast_intervals_for_large_samples(monkeypatch): + seen = {"bca": 0, "fast": 0} + + def fake_bca(frame, max_pairs=20): + seen["bca"] += 1 + return {"available": True} + + def fake_fast(frame, max_pairs=20): + seen["fast"] += 1 + seen["rows"] = len(frame) + seen["pairs"] = max_pairs + return {"available": True} + + monkeypatch.setattr( + "framevitals.budgeted_analysis.run_deep_statistics_v2", + fake_bca, + ) + monkeypatch.setattr( + "framevitals.budgeted_analysis.run_fast_deep_statistics_v2", + fake_fast, + ) + frame = pd.DataFrame({"x": np.arange(5_000), "y": np.arange(5_000)}) + budget = derive_execution_budget(len(frame), len(frame.columns), mode="research") + + result = run_budgeted_deep_statistics(frame, budget=budget) + + assert seen["bca"] == 0 + assert seen["fast"] == 1 + assert seen["rows"] == len(frame) + assert seen["pairs"] <= budget.relationship_pair_budget + assert result["execution"]["inference_strategy"] == ( + "adaptive_large_sample_closed_form_and_order_statistics" + ) + + +def test_anomaly_adapter_discloses_sample_coverage(monkeypatch): + seen = {} + + def fake_anomalies(frame, **kwargs): + seen["rows"] = len(frame) + return {"available": True, "n_rows_scored": len(frame)} + + monkeypatch.setattr( + "framevitals.budgeted_analysis.fast_anomaly_scan", + fake_anomalies, + ) + frame = pd.DataFrame({"x": np.arange(20_000)}) + budget = derive_execution_budget(len(frame), 1, mode="standard") + + result = run_budgeted_anomalies(frame, budget=budget) + + assert seen["rows"] == budget.anomaly_sample_rows + assert result["execution"]["coverage"] == "sample" + assert result["execution"]["sample_rows"] == budget.anomaly_sample_rows + assert result["execution"]["anomaly_strategy"] == "fast_robust_random_projection" + + +def test_research_anomaly_adapter_keeps_heavy_confirmation(monkeypatch): + seen = {"classical": 0, "neural": 0} + + def fake_classical(frame, **kwargs): + seen["classical"] += 1 + return {"available": True} + + def fake_neural(frame, **kwargs): + seen["neural"] += 1 + return {"available": True} + + monkeypatch.setattr( + "framevitals.budgeted_analysis.detect_anomalies_ensemble", + fake_classical, + ) + monkeypatch.setattr( + "framevitals.budgeted_analysis.neural_reconstruction_anomalies", + fake_neural, + ) + frame = pd.DataFrame({"x": np.arange(200), "y": np.arange(200) * 2}) + budget = derive_execution_budget(len(frame), 2, mode="research") + + result = run_budgeted_anomalies(frame, budget=budget) + + assert seen == {"classical": 1, "neural": 1} + assert result["execution"]["anomaly_strategy"] == "classical_ensemble_plus_neural_reconstruction" + assert result["execution"]["neural_reconstruction_enabled"] is True + + +def test_time_series_adapter_preserves_order(monkeypatch): + seen = {} + + def fake_time_series(frame, target_column=None): + seen["values"] = frame["x"].tolist() + return {"available": True} + + monkeypatch.setattr( + "framevitals.budgeted_analysis.detect_and_analyze_time_series", + fake_time_series, + ) + frame = pd.DataFrame({"x": np.arange(20_000)}) + budget = derive_execution_budget(len(frame), 1, mode="standard") + + result = run_budgeted_time_series(frame, budget=budget) + + assert seen["values"] == sorted(seen["values"]) + assert result["execution"]["temporal_order_preserved"] is True + assert result["execution"]["sample_rows"] == budget.time_series_sample_rows + + +def test_numeric_column_state_empty_values_are_safe(): + state = NumericColumnState.from_series(pd.Series([np.nan, np.inf, -np.inf])) + + assert state.count == 0 + assert state.missing == 1 + assert state.infinite == 2 + assert state.mean == 0.0 + assert state.variance is None diff --git a/tests/test_ai_agent.py b/tests/test_ai_agent.py index 10f0714..b061ecb 100644 --- a/tests/test_ai_agent.py +++ b/tests/test_ai_agent.py @@ -1,9 +1,10 @@ import pandas as pd +import pytest + +pytest.importorskip("pydantic") import framevitals.ai_agent as ai_agent -from framevitals.ai_agent import ( - answer_with_agent, -) +from framevitals.ai_agent import answer_with_agent def make_result(): diff --git a/tests/test_analysis_modes.py b/tests/test_analysis_modes.py new file mode 100644 index 0000000..10cf218 --- /dev/null +++ b/tests/test_analysis_modes.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +import framevitals.analysis_api as analysis_api +from framevitals.analysis_selector import select_analyses + + +@pytest.mark.parametrize( + ("mode", "expected_mode_disabled"), + [ + ( + "quick", + { + "deep_statistics", + "anomaly_detection", + "time_series", + "text_profile", + "modeling", + "explainability", + }, + ), + ( + "standard", + {"deep_statistics", "text_profile", "modeling", "explainability"}, + ), + ("deep", {"modeling", "explainability"}), + ("research", set()), + ], +) +def test_mode_policy_is_explicit_and_stable(mode, expected_mode_disabled): + assert set(analysis_api._MODE_DISABLED_MODULES[mode]) == expected_mode_disabled + + +def test_effective_mode_policy_preserves_explicit_user_disables(): + disabled = analysis_api._effective_disabled_modules( + "standard", + ("time_series", "charts"), + ) + assert set(disabled) == { + "deep_statistics", + "text_profile", + "modeling", + "explainability", + "time_series", + "charts", + } + + +def test_quick_keeps_target_intelligence_and_cleaning_available(monkeypatch): + captured: dict[str, object] = {} + + def fake_run_full_analysis(**kwargs): + captured.update(kwargs) + return { + "filename": "", + "profile": {"shape": {"rows": 3, "columns": 2}}, + "execution": {}, + } + + monkeypatch.setattr(analysis_api, "run_full_analysis", fake_run_full_analysis) + result = analysis_api.analyze( + pd.DataFrame({"x": [1, 2, 3], "target": [0, 1, 0]}), + mode="quick", + target="target", + artifacts=True, + ) + + disabled = set(captured["disabled_modules"]) + assert "target_intelligence" not in disabled + assert "cleaning" not in disabled + assert { + "deep_statistics", + "anomaly_detection", + "time_series", + "text_profile", + "modeling", + "explainability", + } <= disabled + assert result["config"] == { + "mode": "quick", + "target": "target", + "artifacts": True, + "workers": result["config"]["workers"], + "disabled_modules": (), + } + + +def test_standard_public_analysis_does_not_schedule_deep_only_modules(monkeypatch): + captured: dict[str, object] = {} + + def fake_run_full_analysis(**kwargs): + captured.update(kwargs) + return { + "filename": "", + "profile": {"shape": {"rows": 3, "columns": 2}}, + "execution": {}, + } + + monkeypatch.setattr(analysis_api, "run_full_analysis", fake_run_full_analysis) + result = analysis_api.analyze( + pd.DataFrame({"x": [1, 2, 3], "target": [0, 1, 0]}), + mode="standard", + target="target", + artifacts=False, + ) + + disabled = set(captured["disabled_modules"]) + assert {"deep_statistics", "text_profile", "modeling", "explainability"} <= disabled + assert "anomaly_detection" not in disabled + assert "time_series" not in disabled + assert "target_intelligence" not in disabled + assert result["config"]["mode"] == "standard" + assert result["config"]["disabled_modules"] == () + + +def test_deep_keeps_advanced_diagnostics_but_reserves_modeling_for_research(monkeypatch): + captured: dict[str, object] = {} + + def fake_run_full_analysis(**kwargs): + captured.update(kwargs) + return { + "filename": "", + "profile": {"shape": {"rows": 3, "columns": 2}}, + "execution": {}, + } + + monkeypatch.setattr(analysis_api, "run_full_analysis", fake_run_full_analysis) + analysis_api.analyze( + pd.DataFrame({"x": [1, 2, 3], "target": [0, 1, 0]}), + mode="deep", + target="target", + artifacts=False, + ) + + disabled = set(captured["disabled_modules"]) + assert disabled == {"modeling", "explainability"} + assert "deep_statistics" not in disabled + assert "text_profile" not in disabled + + +def test_research_keeps_modeling_and_explainability_enabled(monkeypatch): + captured: dict[str, object] = {} + + def fake_run_full_analysis(**kwargs): + captured.update(kwargs) + return { + "filename": "", + "profile": {"shape": {"rows": 3, "columns": 2}}, + "execution": {}, + } + + monkeypatch.setattr(analysis_api, "run_full_analysis", fake_run_full_analysis) + analysis_api.analyze( + pd.DataFrame({"x": [1, 2, 3], "target": [0, 1, 0]}), + mode="research", + target="target", + artifacts=False, + ) + + assert tuple(captured["disabled_modules"]) == () + + +def test_selector_matches_standard_deep_and_research_contracts(): + signals = { + "row_count": 8_000, + "has_numeric_columns": True, + "has_multiple_numeric_columns": True, + "has_categorical_columns": True, + "has_long_text_columns": True, + "has_datetime_columns": True, + "has_time_series_structure": True, + "has_id_like_columns": True, + "has_high_missingness": True, + "has_sensitive_column_candidates": True, + "has_email_like_columns": True, + } + + standard = select_analyses(signals, analysis_mode="standard", target_column="target") + deep = select_analyses(signals, analysis_mode="deep", target_column="target") + research = select_analyses(signals, analysis_mode="research", target_column="target") + + standard_ids = {item["id"] for item in standard["selected_analyses"]} + deep_ids = {item["id"] for item in deep["selected_analyses"]} + research_ids = {item["id"] for item in research["selected_analyses"]} + + assert {"target_analysis", "time_series_signal"} <= standard_ids + assert {"normality_tests", "chi_square_analysis", "text_analysis"} <= deep_ids + assert {"feature_importance", "baseline_model"} <= research_ids + + assert {"normality_tests", "chi_square_analysis", "text_analysis"}.isdisjoint(standard_ids) + assert {"feature_importance", "baseline_model"}.isdisjoint(deep_ids) diff --git a/tests/test_anderson_compatibility.py b/tests/test_anderson_compatibility.py new file mode 100644 index 0000000..0571f53 --- /dev/null +++ b/tests/test_anderson_compatibility.py @@ -0,0 +1,88 @@ +import inspect +from types import SimpleNamespace +import warnings + +import numpy as np +import pandas as pd +import pytest + +from framevitals import deep_statistics_v2 +from framevitals.deep_statistics_v2 import _anderson_normality, _normality + + +def test_anderson_supported_api_is_warning_free(): + rng = np.random.default_rng(42) + sample = pd.Series(rng.normal(size=256)) + + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + result = _normality(sample) + + anderson = result["anderson"] + assert anderson["statistic"] is not None + + if "method" in inspect.signature(deep_statistics_v2.stats.anderson).parameters: + assert anderson["method"] == "interpolate" + assert anderson["critical_5pct"] is None + assert 0.0 <= anderson["p_value"] <= 1.0 + else: + assert anderson["method"] == "legacy_critical_values" + assert anderson["p_value"] is None + assert anderson["critical_5pct"] is not None + + +def test_anderson_legacy_api_falls_back_to_critical_values(monkeypatch): + calls = [] + + def fake_anderson(values, dist="norm", **kwargs): + calls.append(kwargs) + if "method" in kwargs: + raise TypeError("anderson() got an unexpected keyword argument 'method'") + return SimpleNamespace( + statistic=0.42, + critical_values=np.array([0.5, 0.6, 0.7, 0.8, 0.9]), + significance_level=np.array([15.0, 10.0, 5.0, 2.5, 1.0]), + ) + + monkeypatch.setattr(deep_statistics_v2.stats, "anderson", fake_anderson) + result = _anderson_normality(pd.Series(np.arange(20, dtype=float))) + + assert calls == [{"method": "interpolate"}, {}] + assert result == { + "statistic": 0.42, + "p_value": None, + "critical_5pct": 0.7, + "method": "legacy_critical_values", + } + + +def test_normality_verdict_still_prefers_shapiro(monkeypatch): + sample = pd.Series(np.linspace(-2.0, 2.0, 100)) + + monkeypatch.setattr( + deep_statistics_v2.stats, + "shapiro", + lambda values: (0.99, 0.8), + ) + monkeypatch.setattr( + deep_statistics_v2.stats, + "normaltest", + lambda values: (20.0, 0.001), + ) + monkeypatch.setattr( + deep_statistics_v2, + "_anderson_normality", + lambda values: { + "statistic": 2.0, + "p_value": 0.001, + "critical_5pct": None, + "method": "interpolate", + }, + ) + + result = _normality(sample) + + assert result["shapiro"]["p_value"] == pytest.approx(0.8) + assert result["dagostino"]["p_value"] == pytest.approx(0.001) + assert result["anderson"]["p_value"] == pytest.approx(0.001) + assert result["is_probably_normal"] is True diff --git a/tests/test_anomaly_diagnostics_v2.py b/tests/test_anomaly_diagnostics_v2.py new file mode 100644 index 0000000..f9dbef1 --- /dev/null +++ b/tests/test_anomaly_diagnostics_v2.py @@ -0,0 +1,80 @@ +import numpy as np +import pandas as pd +import pytest + +from framevitals.anomaly_ensemble import detect_anomalies_ensemble + + +def _anomaly_frame() -> pd.DataFrame: + rng = np.random.default_rng(42) + rows = 80 + x = rng.normal(0, 1, rows) + y = rng.normal(0, 1, rows) + x[-1] = 12.0 + y[-1] = -10.0 + x[5] = np.inf + y[7] = np.nan + return pd.DataFrame({"x": x, "y": y}) + + +def test_anomaly_ensemble_handles_infinity_and_reports_preparation(): + result = detect_anomalies_ensemble(_anomaly_frame(), top_k=5) + + assert result["available"] is True + assert result["n_rows_scored"] == 80 + assert result["preparation"]["infinite_values_replaced"]["x"] == 1 + assert result["preparation"]["missing_values_imputed"]["x"] == 1 + assert result["preparation"]["missing_values_imputed"]["y"] == 1 + assert result["detectors_run"] + assert isinstance(result["detectors_failed"], dict) + assert isinstance(result["detectors_skipped"], dict) + + +def test_anomaly_ensemble_reports_detector_agreement_and_feature_context(): + result = detect_anomalies_ensemble( + _anomaly_frame(), + contamination=0.05, + threshold=0.5, + top_k=5, + ) + + assert result["available"] is True + assert result["consensus"]["majority_detectors_required"] >= 1 + assert 0 <= result["consensus"]["flagged_fraction"] <= 1 + assert 0 <= result["flagged_fraction"] <= 1 + assert result["expected_anomaly_count"] == 4 + + top = result["top_rows"][0] + assert "agreement_count" in top + assert "agreement_fraction" in top + assert "flagged" in top + assert top["top_feature_deviations"] + assert { + item["feature"] for item in top["top_feature_deviations"] + }.issubset({"x", "y"}) + + for detector in result["detectors_run"]: + assert detector in result["detector_summaries"] + assert detector in result["detector_vote_thresholds"] + + +def test_anomaly_ensemble_keeps_historical_score_fields(): + result = detect_anomalies_ensemble(_anomaly_frame()) + + assert result["threshold"] == 0.6 + assert result["contamination"] == 0.05 + assert "flagged_count" in result + assert "ensemble_summary" in result + assert "top_rows" in result + assert all("ensemble" in row for row in result["top_rows"]) + + +def test_anomaly_ensemble_validates_public_controls(): + frame = _anomaly_frame() + + with pytest.raises(ValueError, match="threshold"): + detect_anomalies_ensemble(frame, threshold=1.1) + with pytest.raises(ValueError, match="top_k"): + detect_anomalies_ensemble(frame, top_k=0) + with pytest.raises(ValueError, match="max_columns"): + detect_anomalies_ensemble(frame, max_columns=0) diff --git a/tests/test_api_facade.py b/tests/test_api_facade.py new file mode 100644 index 0000000..404eef1 --- /dev/null +++ b/tests/test_api_facade.py @@ -0,0 +1,70 @@ +import os +import subprocess +import sys +from pathlib import Path + + +def test_compat_api_import_does_not_eagerly_load_data_stack(): + src_root = Path(__file__).resolve().parents[1] / "src" + env = os.environ.copy() + env["PYTHONPATH"] = str(src_root) + completed = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; import framevitals.api; " + "assert 'pandas' not in sys.modules; " + "assert 'framevitals.pipeline' not in sys.modules; " + "assert 'framevitals.anomaly_ensemble' not in sys.modules" + ), + ], + cwd=src_root.parent, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + + +def test_compat_api_delegates_compare_to_operations(monkeypatch): + import framevitals.api as api + import framevitals.operations as operations + + expected = {"available": True, "marker": "canonical-compare"} + + def fake_compare(reference, current, *, columns=None, max_columns=30): + assert reference == "reference" + assert current == "current" + assert columns == ["value"] + assert max_columns == 4 + return expected + + monkeypatch.setattr(operations, "compare", fake_compare) + result = api.compare( + "reference", + "current", + columns=["value"], + max_columns=4, + ) + + assert result is expected + + +def test_compat_api_delegates_analyze_to_source_dispatcher(monkeypatch): + import framevitals.analysis_api as analysis_api + import framevitals.api as api + + expected = {"marker": "canonical-analysis"} + + def fake_analyze(data, **kwargs): + assert data == "dataset.csv" + assert kwargs["mode"] == "quick" + assert kwargs["artifacts"] is False + return expected + + monkeypatch.setattr(analysis_api, "analyze", fake_analyze) + result = api.analyze("dataset.csv", mode="quick", artifacts=False) + + assert result is expected diff --git a/tests/test_arrow_memory.py b/tests/test_arrow_memory.py new file mode 100644 index 0000000..ceed2dc --- /dev/null +++ b/tests/test_arrow_memory.py @@ -0,0 +1,220 @@ +import numpy as np +import pytest + +pa = pytest.importorskip("pyarrow") + +import framevitals +from framevitals.sources import ArrowTableSource, resolve_source +from framevitals.streaming_profile import ( + STREAM_BATCH_CELL_BUDGET, + STREAM_BATCH_SIZE, + STREAM_SAMPLE_CELL_BUDGET, + STREAM_SAMPLE_ROWS, + _width_aware_row_limit, +) + + +def _table(rows: int = 6_000): + values = np.arange(rows, dtype=np.float64) + values[::127] = np.nan + return pa.table({ + "value": values, + "other": np.arange(rows, dtype=np.float64) * 2.0, + "group": [f"g-{index % 5}" for index in range(rows)], + }) + + +def test_arrow_table_source_exposes_exact_metadata_projection_and_batches(): + table = _table(2_500) + source = resolve_source(table) + + assert isinstance(source, ArrowTableSource) + metadata = source.inspect() + assert metadata.name == "" + assert metadata.kind == "memory" + assert metadata.format == "arrow" + assert metadata.rows == table.num_rows + assert metadata.columns == table.num_columns + assert metadata.size_bytes == table.nbytes + assert metadata.materialized is True + assert metadata.supports_projection is True + assert metadata.supports_streaming is True + assert source.schema() == table.schema + + public_info = framevitals.inspect_source(table) + assert public_info == metadata.to_dict() + + batches = list(source.iter_batches(batch_size=700, columns=["value", "group"])) + assert sum(batch.num_rows for batch in batches) == table.num_rows + assert max(batch.num_rows for batch in batches) <= 700 + assert batches[0].schema.names == ["value", "group"] + + +def test_public_profile_streams_arrow_table_without_pandas_materialization(monkeypatch): + table = _table(12_000) + + def fail_load(self): + raise AssertionError("Arrow profile must not materialize the complete table in pandas") + + monkeypatch.setattr(ArrowTableSource, "load", fail_load) + result = framevitals.profile(table) + + assert result["dataset_name"] == "" + assert result["shape"] == {"rows": table.num_rows, "columns": table.num_columns} + assert result["streaming_metadata"]["enabled"] is True + assert result["streaming_metadata"]["full_materialization"] is False + assert result["source_metadata"]["kind"] == "memory" + assert result["source_metadata"]["format"] == "arrow" + assert result["missing_counts"]["value"] == int(np.isnan(table["value"].to_numpy()).sum()) + + +def test_numpy_streaming_profile_uses_full_stream_quantile_sketch_when_affordable(monkeypatch): + table = _table(12_000) + monkeypatch.setattr("framevitals.streaming_profile.resolve_numeric_backend", lambda: "numpy") + + result = framevitals.profile(table) + numeric = result["numeric_summary_metadata"] + streaming = result["streaming_metadata"] + + assert numeric["backend"] == "numpy" + assert numeric["quantile_source"] == "full_stream_sketch" + assert numeric["quantile_relative_accuracy"] == 0.01 + assert streaming["numpy_full_stream_quantile_sketches"] is True + assert result["numeric_summary"]["other"]["50%"] == pytest.approx(11_999, rel=0.03) + + +def test_numpy_streaming_profile_can_fall_back_to_bounded_sample_quantiles(monkeypatch): + table = _table(12_000) + monkeypatch.setattr("framevitals.streaming_profile.resolve_numeric_backend", lambda: "numpy") + monkeypatch.setattr( + "framevitals.streaming_profile.should_use_full_stream_numpy_sketch", + lambda rows, columns: False, + ) + + result = framevitals.profile(table) + numeric = result["numeric_summary_metadata"] + + assert numeric["quantile_source"] == "bounded_row_sample" + assert numeric["quantile_sketch_skipped_for_cost"] is True + assert result["streaming_metadata"]["numpy_full_stream_quantile_sketches"] is False + + +def test_public_analyze_dispatches_arrow_table_through_streaming_pipeline(monkeypatch): + table = _table(6_000) + + def fail_load(self): + raise AssertionError("Arrow analysis must not materialize the complete table in pandas") + + monkeypatch.setattr(ArrowTableSource, "load", fail_load) + result = framevitals.analyze(table, mode="quick", artifacts=False, workers=1) + + assert result["filename"] == "" + assert result["profile"]["shape"] == { + "rows": table.num_rows, + "columns": table.num_columns, + } + assert result["execution"]["streaming"]["enabled"] is True + assert result["execution"]["streaming"]["full_materialization"] is False + assert result["execution"]["streaming"]["source_rows"] == table.num_rows + + +def test_arrow_record_batch_normalizes_to_streaming_source(): + batch = pa.record_batch({ + "value": [1.0, 2.0, 3.0], + "label": ["a", "b", "a"], + }) + source = resolve_source(batch) + metadata = source.inspect() + + assert isinstance(source, ArrowTableSource) + assert metadata.name == "" + assert metadata.rows == 3 + assert metadata.columns == 2 + assert metadata.supports_streaming is True + + result = framevitals.profile(batch) + assert result["dataset_name"] == "" + assert result["shape"] == {"rows": 3, "columns": 2} + + +def test_arrow_capsule_table_normalizes_without_library_specific_adapter(): + table = _table(1_200) + + class CapsuleTable: + def __arrow_c_stream__(self, requested_schema=None): + return table.__arrow_c_stream__(requested_schema) + + source = resolve_source(CapsuleTable()) + metadata = source.inspect() + + assert isinstance(source, ArrowTableSource) + assert metadata.name == "" + assert metadata.rows == table.num_rows + assert metadata.columns == table.num_columns + assert metadata.supports_streaming is True + + public_info = framevitals.inspect_source(CapsuleTable()) + assert public_info["format"] == "arrow" + assert public_info["rows"] == table.num_rows + assert public_info["supports_streaming"] is True + + result = framevitals.profile(CapsuleTable()) + assert result["shape"] == {"rows": table.num_rows, "columns": table.num_columns} + assert result["source_metadata"]["format"] == "arrow" + + +def test_arrow_record_batch_reader_is_not_silently_materialized(): + table = _table(20) + reader = pa.RecordBatchReader.from_batches(table.schema, table.to_batches()) + + with pytest.raises(TypeError, match="cheap exact row count"): + resolve_source(reader) + + +def test_width_aware_limits_preserve_normal_width_defaults(): + columns = 105 + + assert _width_aware_row_limit( + STREAM_BATCH_SIZE, + columns, + cell_budget=STREAM_BATCH_CELL_BUDGET, + ) == STREAM_BATCH_SIZE + assert _width_aware_row_limit( + STREAM_SAMPLE_ROWS, + columns, + cell_budget=STREAM_SAMPLE_CELL_BUDGET, + ) == STREAM_SAMPLE_ROWS + + +def test_width_aware_limits_clamp_ten_thousand_columns(): + columns = 10_000 + + assert _width_aware_row_limit( + STREAM_BATCH_SIZE, + columns, + cell_budget=STREAM_BATCH_CELL_BUDGET, + ) == 3_200 + assert _width_aware_row_limit( + STREAM_SAMPLE_ROWS, + columns, + cell_budget=STREAM_SAMPLE_CELL_BUDGET, + ) == 600 + + +def test_public_profile_discloses_width_limited_execution(): + rows = 100 + columns = 500 + values = np.arange(rows, dtype=np.float64) + table = pa.table({f"c{index}": values + index for index in range(columns)}) + + result = framevitals.profile(table) + streaming = result["streaming_metadata"] + + assert result["shape"] == {"rows": rows, "columns": columns} + assert streaming["full_materialization"] is False + assert streaming["width_limited"] is True + assert streaming["requested_batch_size"] == STREAM_BATCH_SIZE + assert streaming["batch_size"] == STREAM_BATCH_CELL_BUDGET // columns + assert streaming["requested_sample_rows"] == STREAM_SAMPLE_ROWS + assert streaming["sample_row_limit"] == STREAM_SAMPLE_CELL_BUDGET // columns + assert streaming["sample_cells_retained"] == rows * columns diff --git a/tests/test_backends.py b/tests/test_backends.py new file mode 100644 index 0000000..533db46 --- /dev/null +++ b/tests/test_backends.py @@ -0,0 +1,68 @@ +import numpy as np +import pandas as pd +import pytest + +from framevitals import backends +from framevitals.analysis_state import NumericColumnState + + +def test_numpy_numeric_state_matches_reference_semantics(): + payload = backends.numeric_state( + np.array([1.0, 2.0, np.nan, np.inf, 4.0], dtype=np.float64), + backend="numpy", + ) + + assert payload["backend"] == "numpy" + assert payload["observations"] == 5 + assert payload["count"] == 3 + assert payload["missing"] == 1 + assert payload["infinite"] == 1 + assert payload["mean"] == pytest.approx(7.0 / 3.0) + assert payload["variance"] == pytest.approx(7.0 / 3.0) + assert payload["minimum"] == 1.0 + assert payload["maximum"] == 4.0 + + +def test_auto_backend_falls_back_without_native_extension(monkeypatch): + monkeypatch.setattr(backends, "native_available", lambda: False) + assert backends.resolve_numeric_backend("auto") == "numpy" + + +def test_explicit_rust_backend_requires_native_extension(monkeypatch): + monkeypatch.setattr(backends, "native_available", lambda: False) + with pytest.raises(RuntimeError, match="Rust backend was requested"): + backends.resolve_numeric_backend("rust") + + +def test_backend_environment_override(monkeypatch): + monkeypatch.setenv("FRAMEVITALS_BACKEND", "numpy") + assert backends.resolve_numeric_backend() == "numpy" + + +def test_numeric_profile_numpy_discloses_missing_sketch_layer(): + payload = backends.numeric_profile([1.0, 2.0, 3.0], backend="numpy") + assert payload["backend"] == "numpy" + assert payload["sketches_available"] is False + + +def test_analysis_state_reconstructs_m2_from_backend_variance(monkeypatch): + monkeypatch.setattr( + "framevitals.analysis_state.numeric_state", + lambda series: { + "count": 3, + "missing": 1, + "infinite": 0, + "mean": 2.0, + "variance": 1.0, + "std": 1.0, + "minimum": 1.0, + "maximum": 3.0, + }, + ) + state = NumericColumnState.from_series(pd.Series([1.0, 2.0, 3.0, None])) + + assert state.count == 3 + assert state.missing == 1 + assert state.mean == 2.0 + assert state.m2 == 2.0 + assert state.variance == 1.0 diff --git a/tests/test_benchmark_harness.py b/tests/test_benchmark_harness.py new file mode 100644 index 0000000..1c1a832 --- /dev/null +++ b/tests/test_benchmark_harness.py @@ -0,0 +1,233 @@ +import json +import math +from pathlib import Path +import statistics +import subprocess +import sys + + +def test_scale_benchmark_harness_emits_machine_readable_result(tmp_path): + output = tmp_path / "benchmark.json" + script = Path("benchmarks/benchmark_profile_scale.py") + + completed = subprocess.run( + [ + sys.executable, + str(script), + "--rows", + "200", + "--numeric-columns", + "4", + "--categorical-columns", + "1", + "--scenarios", + "numpy", + "--output", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["benchmark_schema_version"] == 1 + assert payload["workload"]["rows"] == 200 + assert payload["workload"]["total_columns"] == 5 + assert len(payload["measurements"]) == 1 + + measurement = payload["measurements"][0] + assert measurement["scenario"] == "numpy" + assert measurement["elapsed_seconds"] >= 0 + assert measurement["peak_rss_mb"] > 0 + assert measurement["result"]["shape"] == {"rows": 200, "columns": 5} + assert json.loads(completed.stdout)["workload"] == payload["workload"] + + +def test_release_delta_benchmark_evidence_is_self_consistent(): + evidence = Path( + "benchmarks/results/release_0.2.0_vs_0.1.0_10k_x64.json" + ) + payload = json.loads(evidence.read_text(encoding="utf-8")) + + assert payload["benchmark_schema_version"] == 1 + assert payload["comparison"] == "FrameVitals 0.2.0 vs 0.1.0" + assert payload["old_ref"] == ( + "v0.1.0@3da1432168fbfcb3dbe99fcfb6f6200f5e63214b" + ) + assert payload["new_ref"] == ( + "develop/august@05b11e594995a3833ec08f5b4a8a145197bf4cab" + ) + + dataset = payload["dataset"] + assert dataset["rows"] == 10_000 + assert dataset["columns"] == 64 + assert dataset["cells"] == 640_000 + assert dataset["format"] == "csv" + + methodology = payload["methodology"] + assert methodology["warmups_per_version_mode"] == 1 + assert methodology["measured_repetitions_per_version_mode"] == 3 + assert methodology["measurement_order"] == "ABBAAB" + assert methodology["scope"] == ( + "all 10,000 rows and all 64 columns in both releases" + ) + + measurements = payload["measurements"] + assert len(measurements) == 12 + assert {(item["version"], item["mode"]) for item in measurements} == { + ("0.1.0", "quick"), + ("0.2.0", "quick"), + ("0.1.0", "standard"), + ("0.2.0", "standard"), + } + + for mode in ("quick", "standard"): + mode_result = payload["modes"][mode] + old = mode_result["0.1.0"] + new = mode_result["0.2.0"] + + for version_result in (old, new): + assert version_result["profiled_columns"] == 64 + assert len(version_result["wall_seconds"]) == 3 + assert len(version_result["peak_rss_mb"]) == 3 + assert math.isclose( + version_result["median_wall_seconds"], + statistics.median(version_result["wall_seconds"]), + rel_tol=0.0, + abs_tol=1e-12, + ) + assert math.isclose( + version_result["median_peak_rss_mb"], + statistics.median(version_result["peak_rss_mb"]), + rel_tol=0.0, + abs_tol=1e-12, + ) + + assert old["backend"]["selected"] == "legacy-python" + assert old["backend"]["native_available"] is False + assert new["backend"]["selected"] == "rust" + assert new["backend"]["native_available"] is True + + expected_speedup = ( + old["median_wall_seconds"] / new["median_wall_seconds"] + ) + expected_wall_reduction = ( + 1.0 + - new["median_wall_seconds"] / old["median_wall_seconds"] + ) * 100.0 + expected_rss_reduction = ( + 1.0 + - new["median_peak_rss_mb"] / old["median_peak_rss_mb"] + ) * 100.0 + + assert math.isclose( + mode_result["speedup_x"], + expected_speedup, + rel_tol=1e-12, + ) + assert math.isclose( + mode_result["wall_time_reduction_percent"], + expected_wall_reduction, + rel_tol=1e-12, + ) + assert math.isclose( + mode_result["peak_rss_reduction_percent"], + expected_rss_reduction, + rel_tol=1e-12, + ) + + assert new["median_wall_seconds"] < old["median_wall_seconds"] + + +def test_release_accuracy_evidence_matches_same_performance_dataset_and_contract(): + evidence = Path( + "benchmarks/results/release_0.2.0_vs_0.1.0_accuracy_10k_x64.json" + ) + payload = json.loads(evidence.read_text(encoding="utf-8")) + + assert payload["benchmark_schema_version"] == 1 + assert payload["comparison"] == ( + "FrameVitals 0.2.0 vs 0.1.0 same-dataset statistical accuracy" + ) + assert payload["old_ref"] == ( + "v0.1.0@3da1432168fbfcb3dbe99fcfb6f6200f5e63214b" + ) + + dataset = payload["dataset"] + assert dataset == { + "bytes": 2_811_188, + "cells": 640_000, + "columns": 64, + "format": "csv", + "generator": "((row * (col + 3) + col * 17) % 2001 - 1000).astype(int16)", + "rows": 10_000, + "same_as_release_performance_run": 32010158292, + } + assert payload["tracked_columns"] == [ + "c000", + "c001", + "c007", + "c031", + "c063", + ] + assert payload["evidence_runs"] == { + "accuracy_full_legacy_and_initial_native_run": 32014979365, + "native_shape_corrected_run": 32015665811, + "same_dataset_performance_run": 32010158292, + } + + old = payload["versions"]["0.1.0"] + new = payload["versions"]["0.2.0"] + assert old["backend"] == { + "native_available": False, + "selected": "legacy-python", + } + assert new["backend"]["selected"] == "rust" + assert new["backend"]["native_available"] is True + + old_summary = old["summary"] + new_summary = new["summary"] + unchanged_metrics = [ + "max_exact_fact_absolute_error", + "max_mean_absolute_error", + "max_std_absolute_error", + "max_shape_absolute_error", + "pearson_absolute_error", + ] + for metric in unchanged_metrics: + assert math.isclose( + new_summary[metric], + old_summary[metric], + rel_tol=0.0, + abs_tol=1e-15, + ) + assert math.isclose( + payload["delta_new_minus_old"][metric], + 0.0, + rel_tol=0.0, + abs_tol=1e-15, + ) + + assert old_summary["max_exact_fact_absolute_error"] == 0.0 + assert new_summary["max_exact_fact_absolute_error"] == 0.0 + assert old_summary["shape_values_unavailable"] == 0 + assert new_summary["shape_values_unavailable"] == 0 + + quantiles = payload["quantile_error_context"] + assert quantiles["tracked_quantile_values"] == 15 + assert quantiles["native_quantile_relative_accuracy_setting"] == 0.01 + assert math.isclose( + new_summary["max_quantile_absolute_error"], + quantiles["max_absolute_error_units"], + rel_tol=0.0, + abs_tol=1e-12, + ) + assert math.isclose( + new_summary["mean_quantile_absolute_error"], + quantiles["mean_absolute_error_units"], + rel_tol=0.0, + abs_tol=1e-12, + ) + assert quantiles["max_error_percent_of_column_range"] < 0.211 + assert quantiles["mean_error_percent_of_column_range"] < 0.080 diff --git a/tests/test_check_plugins.py b/tests/test_check_plugins.py new file mode 100644 index 0000000..bb3024d --- /dev/null +++ b/tests/test_check_plugins.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +import framevitals as fv +from framevitals.checks import DataCheck +from framevitals.plugins import CHECK_ENTRYPOINT_GROUP, discover_checks + + +class _FakeEntryPoint: + def __init__(self, name, value, loaded=None, error=None): + self.name = name + self.value = value + self._loaded = loaded + self._error = error + + def load(self): + if self._error is not None: + raise self._error + return self._loaded + + +class _FakeEntryPoints(list): + def select(self, *, group): + assert group == CHECK_ENTRYPOINT_GROUP + return self + + +def test_discover_checks_loads_callables_and_datachecks(monkeypatch): + def positive_values(df: pd.DataFrame): + return bool((df["value"] > 0).all()) + + explicit = DataCheck( + name="explicit check", + function=lambda df: True, + severity="warning", + ) + entries = _FakeEntryPoints([ + _FakeEntryPoint("z_plugin", "pkg.z:check", loaded=explicit), + _FakeEntryPoint("a_plugin", "pkg.a:positive_values", loaded=positive_values), + ]) + monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries) + + checks = discover_checks() + + assert [item.name for item in checks] == ["a_plugin", "explicit check"] + assert checks[0](pd.DataFrame({"value": [1, 2]})) is True + assert checks[1].severity == "warning" + + +def test_public_discover_checks_delegates_to_opt_in_plugin_loader(monkeypatch): + def positive_values(df: pd.DataFrame): + return bool((df["value"] > 0).all()) + + entries = _FakeEntryPoints([ + _FakeEntryPoint("positive_values", "pkg:positive_values", loaded=positive_values), + ]) + monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries) + + checks = fv.discover_checks() + + assert len(checks) == 1 + assert isinstance(checks[0], fv.DataCheck) + assert checks[0].name == "positive_values" + + +def test_discover_checks_rejects_duplicate_public_names(monkeypatch): + first = DataCheck(name="same", function=lambda df: True) + second = DataCheck(name="same", function=lambda df: True) + entries = _FakeEntryPoints([ + _FakeEntryPoint("one", "pkg.one:check", loaded=first), + _FakeEntryPoint("two", "pkg.two:check", loaded=second), + ]) + monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries) + + with pytest.raises(ValueError, match="Duplicate FrameVitals check plugin name"): + discover_checks() + + +def test_discover_checks_surfaces_plugin_load_failures(monkeypatch): + entries = _FakeEntryPoints([ + _FakeEntryPoint( + "broken", + "broken_pkg:check", + error=ImportError("provider dependency missing"), + ) + ]) + monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries) + + with pytest.raises(RuntimeError, match="broken"): + discover_checks() + + +def test_discover_checks_rejects_unsupported_exports(monkeypatch): + entries = _FakeEntryPoints([ + _FakeEntryPoint("bad", "pkg.bad:value", loaded=42), + ]) + monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", lambda: entries) + + with pytest.raises(TypeError, match="DataCheck or DataFrame callable"): + discover_checks() + + +def test_discover_checks_can_target_an_explicit_group(monkeypatch): + class _Groups: + def select(self, *, group): + assert group == "acme.framevitals.checks" + return [] + + monkeypatch.setattr("framevitals.plugins.importlib_metadata.entry_points", _Groups) + + assert discover_checks(group="acme.framevitals.checks") == [] diff --git a/tests/test_cleaning_plans.py b/tests/test_cleaning_plans.py new file mode 100644 index 0000000..2f3e13b --- /dev/null +++ b/tests/test_cleaning_plans.py @@ -0,0 +1,165 @@ +import json + +import pandas as pd + +import framevitals +from framevitals.cleaner import create_cleaned_dataset +from framevitals.cleaning_plan import infer_cleaning_plan +from framevitals.cli import main +from framevitals.health_score import calculate_health_score +from framevitals.profiler import build_profile + + +def _dirty_frame() -> pd.DataFrame: + return pd.DataFrame({ + "age": [20.0, None, 30.0, 30.0], + "city": ["Pune", "Mumbai", None, None], + }) + + +def test_cleaning_plan_is_explicit_and_does_not_mutate_input(): + df = _dirty_frame() + original = df.copy(deep=True) + + plan = infer_cleaning_plan(df) + + pd.testing.assert_frame_equal(df, original) + assert plan["schema_version"] == "1" + assert plan.summary()["action_count"] == 3 + assert plan["duplicates_to_remove"] == 1 + assert plan["missing_values_to_fill"] == 2 + assert [action["type"] for action in plan.actions] == [ + "remove_duplicates", + "fill_numeric_missing", + "fill_categorical_missing", + ] + + +def test_apply_cleaning_plan_returns_clean_copy(): + df = _dirty_frame() + plan = infer_cleaning_plan(df) + + cleaned = plan.apply(df) + + assert cleaned is not df + assert len(cleaned) == 3 + assert int(cleaned.isna().sum().sum()) == 0 + assert cleaned.loc[1, "age"] == 25.0 + assert cleaned.loc[2, "city"] == "Mumbai" + assert pd.isna(df.loc[1, "age"]) + + +def test_cleaning_simulation_reports_expected_improvement(): + df = _dirty_frame() + profile = build_profile(df) + health = calculate_health_score(df, profile) + plan = infer_cleaning_plan(df, profile=profile) + + simulation = plan.simulate( + df, + before_profile=profile, + before_health=health, + ) + + assert simulation["rows_removed"] == 1 + assert simulation["missing_before"] == 3 + assert simulation["missing_after"] == 0 + assert simulation["duplicates_before"] == 1 + assert simulation["duplicates_after"] == 0 + assert simulation["health_delta"] >= 0 + + +def test_internal_cleaner_preserves_legacy_payload_and_adds_structured_plan(): + df = _dirty_frame() + result = create_cleaned_dataset("test", df, write_output=False) + + assert result["actions"] == [ + { + "action": "Remove duplicates", + "details": "Removed 1 duplicate rows.", + "risk": "Low", + }, + { + "action": "Fill numeric missing values", + "details": "Filled 1 missing values in 'age' using median.", + "risk": "Medium", + }, + { + "action": "Fill categorical missing values", + "details": "Filled 1 missing values in 'city' using mode.", + "risk": "Medium", + }, + ] + assert result["missing_before"] == 3 + assert result["missing_after"] == 0 + assert result["duplicates_before"] == 1 + assert result["duplicates_after"] == 0 + assert result["plan"]["schema_version"] == "1" + + +def test_public_cleaning_api_supports_dataframe_and_file(tmp_path): + df = _dirty_frame() + plan = framevitals.plan_cleaning(df) + cleaned = framevitals.clean(df, plan=plan) + + assert isinstance(plan, framevitals.CleaningPlan) + assert len(cleaned) == 3 + assert cleaned.isna().sum().sum() == 0 + + path = tmp_path / "dirty.csv" + df.to_csv(path, index=False) + file_plan = framevitals.plan_cleaning(path) + file_cleaned = framevitals.clean(path, plan=file_plan) + + assert file_plan.summary()["action_count"] == 3 + assert len(file_cleaned) == 3 + assert file_cleaned.isna().sum().sum() == 0 + + +def test_datetime_mode_fill_stays_datetime_dtype(): + df = pd.DataFrame({ + "event_time": [ + pd.Timestamp("2026-01-01"), + pd.NaT, + pd.Timestamp("2026-01-01"), + ], + }) + plan = infer_cleaning_plan(df) + cleaned = plan.apply(df) + + assert pd.api.types.is_datetime64_any_dtype(cleaned["event_time"]) + assert cleaned["event_time"].isna().sum() == 0 + + +def test_cli_clean_is_plan_only_without_output_and_explicit_with_output( + tmp_path, + monkeypatch, + capsys, +): + dataset = tmp_path / "dirty.csv" + cleaned_path = tmp_path / "cleaned.csv" + plan_path = tmp_path / "plan.json" + _dirty_frame().to_csv(dataset, index=False) + + monkeypatch.setattr( + "sys.argv", + ["framevitals", "clean", str(dataset), "--plan-output", str(plan_path)], + ) + assert main() == 0 + plan_only_payload = json.loads(capsys.readouterr().out) + assert plan_only_payload["cleaned_output"] is None + assert not cleaned_path.exists() + assert plan_path.exists() + + monkeypatch.setattr( + "sys.argv", + ["framevitals", "clean", str(dataset), "--output", str(cleaned_path)], + ) + assert main() == 0 + apply_payload = json.loads(capsys.readouterr().out) + assert apply_payload["cleaned_output"] == str(cleaned_path) + assert cleaned_path.exists() + + cleaned = pd.read_csv(cleaned_path) + assert len(cleaned) == 3 + assert cleaned.isna().sum().sum() == 0 diff --git a/tests/test_cli.py b/tests/test_cli.py index f33839f..d7b0afb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,10 +17,70 @@ def test_cli_parser(): assert args.file.name == "dataset.csv" assert args.mode == "quick" assert args.target is None - assert args.artifacts is False + assert args.artifacts is None + assert args.workers is None + assert args.preset is None + assert args.config is None assert args.output is None +def test_cli_inspect_parser(): + parser = build_parser() + + args = parser.parse_args([ + "inspect", + "dataset.parquet", + "--format", + "json", + "--output", + "source.json", + ]) + + assert args.command == "inspect" + assert args.file.name == "dataset.parquet" + assert args.format == "json" + assert args.output.name == "source.json" + + +def test_cli_inspect_emits_source_metadata_and_json_file( + tmp_path, + monkeypatch, + capsys, +): + dataset = tmp_path / "dataset.csv" + output = tmp_path / "source.json" + dataset.write_text("value,label\n1,a\n2,b\n3,c\n", encoding="utf-8") + + monkeypatch.setattr( + "framevitals.sources.DelimitedTextSource._pyarrow_csv", + lambda self: None, + ) + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "inspect", + str(dataset), + "--format", + "json", + "--output", + str(output), + ], + ) + + assert main() == 0 + rendered = json.loads(capsys.readouterr().out) + saved = json.loads(output.read_text(encoding="utf-8")) + + assert rendered == saved + assert rendered["name"] == "dataset.csv" + assert rendered["kind"] == "file" + assert rendered["format"] == "csv" + assert rendered["supports_streaming"] is False + assert rendered["supports_projection"] is False + assert rendered["size_bytes"] == dataset.stat().st_size + + def test_cli_target_argument(): parser = build_parser() @@ -32,6 +92,8 @@ def test_cli_target_argument(): "--mode", "deep", "--artifacts", + "--workers", + "6", "--output", "result.json", ]) @@ -40,10 +102,39 @@ def test_cli_target_argument(): assert args.target == "churn" assert args.mode == "deep" assert args.artifacts is True + assert args.workers == 6 assert args.output.name == "result.json" -def test_cli_compare_parser(): +def test_cli_config_and_preset_arguments(): + parser = build_parser() + + args = parser.parse_args([ + "analyze", + "customers.csv", + "--preset", + "ci", + "--config", + "framevitals.toml", + "--no-artifacts", + ]) + config_args = parser.parse_args([ + "config", + "--file", + "framevitals.toml", + "--preset", + "deep", + ]) + + assert args.preset == "ci" + assert args.config.name == "framevitals.toml" + assert args.artifacts is False + assert config_args.command == "config" + assert config_args.file.name == "framevitals.toml" + assert config_args.preset == "deep" + + +def test_cli_compare_parser_supports_gate_controls(): parser = build_parser() args = parser.parse_args([ @@ -54,6 +145,10 @@ def test_cli_compare_parser(): "age,income", "--max-columns", "12", + "--format", + "terminal", + "--fail-on", + "moderate", "--output", "drift.json", ]) @@ -63,15 +158,25 @@ def test_cli_compare_parser(): assert args.current.name == "production.csv" assert args.columns == "age,income" assert args.max_columns == 12 + assert args.format == "terminal" + assert args.fail_on == "moderate" assert args.output.name == "drift.json" -def test_cli_contract_parsers(): +def test_cli_contract_parsers_expose_inference_and_warning_controls(): parser = build_parser() infer_args = parser.parse_args([ "infer-contract", "reference.csv", + "--numeric-tolerance", + "0.1", + "--max-categories", + "15", + "--null-fraction-tolerance", + "0.08", + "--no-infer-unique", + "--allow-extra-columns", "--output", "contract.json", ]) @@ -80,20 +185,30 @@ def test_cli_contract_parsers(): "candidate.csv", "--contract", "contract.json", + "--format", + "terminal", + "--fail-on-warn", "--output", "validation.json", ]) assert infer_args.command == "infer-contract" assert infer_args.file.name == "reference.csv" + assert infer_args.numeric_tolerance == 0.1 + assert infer_args.max_categories == 15 + assert infer_args.null_fraction_tolerance == 0.08 + assert infer_args.infer_unique is False + assert infer_args.allow_extra_columns is True assert infer_args.output.name == "contract.json" assert validate_args.command == "validate" assert validate_args.file.name == "candidate.csv" assert validate_args.contract.name == "contract.json" + assert validate_args.format == "terminal" + assert validate_args.fail_on_warn is True assert validate_args.output.name == "validation.json" -def test_cli_validate_returns_nonzero_for_contract_errors(tmp_path, monkeypatch): +def test_cli_validate_returns_two_for_contract_errors(tmp_path, monkeypatch): dataset = tmp_path / "candidate.csv" contract = tmp_path / "contract.json" dataset.write_text("age\n15\n", encoding="utf-8") @@ -122,4 +237,44 @@ def test_cli_validate_returns_nonzero_for_contract_errors(tmp_path, monkeypatch) ], ) + assert main() == 2 + + +def test_cli_validate_warning_exit_is_opt_in(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "candidate.csv" + contract = tmp_path / "contract.json" + dataset.write_text("plan\nenterprise\n", encoding="utf-8") + contract.write_text( + json.dumps({ + "version": 2, + "columns": { + "plan": { + "type": "string", + "nullable": False, + "allowed_values": ["basic", "pro"], + "allowed_values_severity": "warning", + }, + }, + }), + encoding="utf-8", + ) + + monkeypatch.setattr( + "sys.argv", + ["framevitals", "validate", str(dataset), "--contract", str(contract)], + ) + assert main() == 0 + capsys.readouterr() + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "validate", + str(dataset), + "--contract", + str(contract), + "--fail-on-warn", + ], + ) assert main() == 1 diff --git a/tests/test_cli_entry_streaming.py b/tests/test_cli_entry_streaming.py new file mode 100644 index 0000000..2ea9c07 --- /dev/null +++ b/tests/test_cli_entry_streaming.py @@ -0,0 +1,52 @@ +import json + +import numpy as np +import pandas as pd +import pytest + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + +from framevitals.cli_entry import main +from framevitals.sources import ParquetSource + + +def test_installed_cli_analyze_streams_parquet(tmp_path, monkeypatch, capsys): + path = tmp_path / "cli-stream.parquet" + frame = pd.DataFrame({ + "value": np.arange(12_000, dtype=np.float64), + "other": np.arange(12_000, dtype=np.float64) * 2.0, + "group": [f"g-{index % 5}" for index in range(12_000)], + }) + pq.write_table( + pa.Table.from_pandas(frame, preserve_index=False), + path, + row_group_size=777, + ) + + def fail_load(self): + raise AssertionError("installed CLI analyze must not materialize full Parquet") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "analyze", + str(path), + "--mode", + "quick", + "--no-artifacts", + "--workers", + "1", + "--format", + "json", + ], + ) + + assert main() == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["profile"]["shape"] == {"rows": len(frame), "columns": 3} + assert payload["execution"]["streaming"]["enabled"] is True + assert payload["execution"]["streaming"]["full_materialization"] is False + assert payload["execution"]["streaming"]["working_sample_rows"] == 5_000 diff --git a/tests/test_cli_gate.py b/tests/test_cli_gate.py new file mode 100644 index 0000000..b801bde --- /dev/null +++ b/tests/test_cli_gate.py @@ -0,0 +1,123 @@ +import json + +from framevitals.cli import build_parser, main + + +def test_gate_parser_exposes_ci_controls(): + parser = build_parser() + args = parser.parse_args([ + "gate", + "production.csv", + "--reference", + "training.csv", + "--contract", + "contract.json", + "--columns", + "age,income", + "--max-columns", + "12", + "--drift-warn-on", + "minor", + "--drift-fail-on", + "moderate", + "--fail-on-validation-warning", + "--format", + "json", + "--output", + "gate.json", + ]) + + assert args.command == "gate" + assert args.current.name == "production.csv" + assert args.reference.name == "training.csv" + assert args.contract.name == "contract.json" + assert args.columns == "age,income" + assert args.max_columns == 12 + assert args.drift_warn_on == "minor" + assert args.drift_fail_on == "moderate" + assert args.fail_on_validation_warning is True + assert args.format == "json" + assert args.output.name == "gate.json" + + +def test_gate_cli_returns_zero_for_passing_contract(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "candidate.csv" + contract = tmp_path / "contract.json" + dataset.write_text("age,plan\n25,basic\n30,pro\n", encoding="utf-8") + contract.write_text( + json.dumps({ + "version": 2, + "allow_extra_columns": False, + "columns": { + "age": { + "type": "integer", + "nullable": False, + "minimum": 18, + "maximum": 100, + }, + "plan": { + "type": "string", + "nullable": False, + "allowed_values": ["basic", "pro"], + }, + }, + }), + encoding="utf-8", + ) + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "gate", + str(dataset), + "--contract", + str(contract), + "--format", + "json", + ], + ) + + assert main() == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["status"] == "pass" + assert payload["passed"] is True + assert payload["checks_run"] == ["validation"] + + +def test_gate_cli_returns_one_for_failing_contract(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "candidate.csv" + contract = tmp_path / "contract.json" + dataset.write_text("age\n15\n", encoding="utf-8") + contract.write_text( + json.dumps({ + "version": 2, + "allow_extra_columns": False, + "columns": { + "age": { + "type": "integer", + "nullable": False, + "minimum": 18, + "maximum": 100, + }, + }, + }), + encoding="utf-8", + ) + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "gate", + str(dataset), + "--contract", + str(contract), + "--format", + "json", + ], + ) + + assert main() == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["status"] == "fail" + assert payload["passed"] is False + assert payload["checks"]["validation"]["status"] == "fail" diff --git a/tests/test_cli_gate_streaming.py b/tests/test_cli_gate_streaming.py new file mode 100644 index 0000000..83e3b2c --- /dev/null +++ b/tests/test_cli_gate_streaming.py @@ -0,0 +1,58 @@ +import json + +import numpy as np +import pandas as pd +import pytest + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + +from framevitals.cli import main +from framevitals.sources import ParquetSource + + +def _write_parquet(path, *, rows: int, shift: float = 0.0) -> None: + frame = pd.DataFrame({ + "value": np.arange(rows, dtype=np.float64) + shift, + "group": [f"g-{index % 5}" for index in range(rows)], + }) + pq.write_table( + pa.Table.from_pandas(frame, preserve_index=False), + path, + row_group_size=777, + ) + + +def test_cli_reference_gate_streams_parquet(tmp_path, monkeypatch, capsys): + reference = tmp_path / "reference.parquet" + current = tmp_path / "current.parquet" + _write_parquet(reference, rows=60_000) + _write_parquet(current, rows=72_000, shift=50.0) + + def fail_load(self): + raise AssertionError("CLI reference gate must not fully materialize Parquet inputs") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "gate", + str(current), + "--reference", + str(reference), + "--format", + "json", + ], + ) + + exit_code = main() + assert exit_code in {0, 1} + payload = json.loads(capsys.readouterr().out) + execution = payload["execution"]["drift"] + assert execution["full_materialization"] is False + assert execution["reference"]["source_rows"] == 60_000 + assert execution["reference"]["sample_rows"] == 50_000 + assert execution["current"]["source_rows"] == 72_000 + assert execution["current"]["sample_rows"] == 50_000 + assert execution["components"]["value_distributions"] == "bounded_row_sample" diff --git a/tests/test_cli_monitoring.py b/tests/test_cli_monitoring.py new file mode 100644 index 0000000..6f04a65 --- /dev/null +++ b/tests/test_cli_monitoring.py @@ -0,0 +1,184 @@ +import json + +from framevitals.cli import build_parser, main +from framevitals.snapshots import load_snapshot + + +def _dataset(path): + path.write_text( + "value,other,group\n" + "1,2,a\n" + "2,4,b\n" + "3,6,a\n" + "4,8,b\n" + "5,10,a\n" + "6,12,b\n", + encoding="utf-8", + ) + + +def test_snapshot_and_compare_snapshot_parsers(): + parser = build_parser() + + snapshot_args = parser.parse_args([ + "snapshot", + "dataset.csv", + "--mode", + "quick", + "--workers", + "1", + "--output", + "snapshot.json", + ]) + assert snapshot_args.command == "snapshot" + assert snapshot_args.file.name == "dataset.csv" + assert snapshot_args.mode == "quick" + assert snapshot_args.workers == 1 + assert snapshot_args.output.name == "snapshot.json" + + compare_args = parser.parse_args([ + "compare-snapshots", + "baseline.json", + "current.json", + "--format", + "json", + "--fail-on-change", + "--output", + "diff.json", + ]) + assert compare_args.command == "compare-snapshots" + assert compare_args.reference.name == "baseline.json" + assert compare_args.current.name == "current.json" + assert compare_args.format == "json" + assert compare_args.fail_on_change is True + assert compare_args.output.name == "diff.json" + + +def test_snapshot_cli_writes_loadable_compact_state(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "dataset.csv" + snapshot_path = tmp_path / "snapshot.json" + _dataset(dataset) + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "snapshot", + str(dataset), + "--mode", + "quick", + "--workers", + "1", + "--format", + "json", + "--output", + str(snapshot_path), + ], + ) + + assert main() == 0 + stdout_payload = json.loads(capsys.readouterr().out) + saved = load_snapshot(snapshot_path) + + assert stdout_payload["snapshot_schema_version"] == "1" + assert stdout_payload["fingerprint"] == saved["fingerprint"] + assert saved["source"]["filename"] == "dataset.csv" + assert saved["state"]["analysis_mode"] == "quick" + assert saved["state"]["config"]["artifacts"] is False + assert "profile" not in saved + + +def test_compare_snapshots_cli_reports_unchanged_state(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "dataset.csv" + snapshot_path = tmp_path / "snapshot.json" + diff_path = tmp_path / "diff.json" + _dataset(dataset) + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "snapshot", + str(dataset), + "--mode", + "quick", + "--workers", + "1", + "--format", + "json", + "--output", + str(snapshot_path), + ], + ) + assert main() == 0 + capsys.readouterr() + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "compare-snapshots", + str(snapshot_path), + str(snapshot_path), + "--format", + "json", + "--fail-on-change", + "--output", + str(diff_path), + ], + ) + assert main() == 0 + diff = json.loads(capsys.readouterr().out) + + assert diff["changed"] is False + assert diff["schema"]["added_columns"] == [] + assert diff["schema"]["removed_columns"] == [] + assert json.loads(diff_path.read_text(encoding="utf-8")) == diff + + +def test_compare_snapshots_cli_can_fail_ci_on_change(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "dataset.csv" + baseline = tmp_path / "baseline.json" + current = tmp_path / "current.json" + _dataset(dataset) + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "snapshot", + str(dataset), + "--mode", + "quick", + "--workers", + "1", + "--output", + str(baseline), + ], + ) + assert main() == 0 + capsys.readouterr() + + payload = json.loads(baseline.read_text(encoding="utf-8")) + payload["fingerprint"] = "0" * 64 + payload["state"]["dataset"]["dtypes"]["new_column"] = "int64" + current.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "compare-snapshots", + str(baseline), + str(current), + "--format", + "terminal", + "--fail-on-change", + ], + ) + assert main() == 1 + rendered = capsys.readouterr().out + + assert "FrameVitals snapshot diff" in rendered + assert "Changed yes" in rendered + assert "Added columns 1" in rendered diff --git a/tests/test_cli_reporting.py b/tests/test_cli_reporting.py new file mode 100644 index 0000000..d428de9 --- /dev/null +++ b/tests/test_cli_reporting.py @@ -0,0 +1,77 @@ +import json + +from framevitals.cli import main + + +def _write_dataset(path): + path.write_text( + "age,city\n" + "20,Pune\n" + "30,Mumbai\n" + "40,Pune\n" + "50,Nashik\n", + encoding="utf-8", + ) + + +def test_analyze_cli_writes_full_json_and_html(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "customers.csv" + output = tmp_path / "report.json" + html = tmp_path / "report.html" + _write_dataset(dataset) + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "analyze", + str(dataset), + "--mode", + "quick", + "--output", + str(output), + "--html-report", + str(html), + ], + ) + + assert main() == 0 + stdout = capsys.readouterr().out + assert "FrameVitals Analysis" in stdout + assert "Full JSON" in stdout + assert "HTML report" in stdout + + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["filename"] == "customers.csv" + assert payload["profile"]["shape"]["rows"] == 4 + assert payload["result_schema_version"] == "1" + assert "findings" in payload + + html_text = html.read_text(encoding="utf-8") + assert "FrameVitals analysis report" in html_text + assert "customers.csv" in html_text + + +def test_analyze_cli_json_stdout_is_complete(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "customers.csv" + _write_dataset(dataset) + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "analyze", + str(dataset), + "--mode", + "quick", + "--format", + "json", + ], + ) + + assert main() == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["profile"]["columns"] == ["age", "city"] + assert "health" in payload + assert "ml_readiness" in payload + assert "findings" in payload diff --git a/tests/test_cli_system_info.py b/tests/test_cli_system_info.py new file mode 100644 index 0000000..1a78a5b --- /dev/null +++ b/tests/test_cli_system_info.py @@ -0,0 +1,85 @@ +import json + +from framevitals.cli import build_parser, main + + +def test_system_info_parser_exposes_gpu_and_output_controls(): + parser = build_parser() + + args = parser.parse_args([ + "system-info", + "--no-probe-gpu", + "--format", + "json", + "--output", + "system.json", + ]) + + assert args.command == "system-info" + assert args.probe_gpu is False + assert args.format == "json" + assert args.output.name == "system.json" + + +def test_system_info_cli_routes_to_canonical_capability_api( + tmp_path, + monkeypatch, + capsys, +): + output = tmp_path / "system.json" + expected = { + "python": "3.12.0", + "backend": "numpy", + "native": { + "available": False, + "reason": "test fixture", + }, + "gpu": { + "probed": False, + "available": False, + }, + } + calls = [] + + def fake_system_info(*, probe_gpu=True): + calls.append(probe_gpu) + return expected + + monkeypatch.setattr("framevitals.acceleration.system_info", fake_system_info) + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "system-info", + "--no-probe-gpu", + "--format", + "json", + "--output", + str(output), + ], + ) + + assert main() == 0 + assert calls == [False] + stdout = json.loads(capsys.readouterr().out) + saved = json.loads(output.read_text(encoding="utf-8")) + assert stdout == expected + assert saved == expected + + +def test_system_info_terminal_render_is_human_readable(monkeypatch, capsys): + monkeypatch.setattr( + "framevitals.acceleration.system_info", + lambda *, probe_gpu=True: { + "backend": "numpy", + "native": {"available": True}, + }, + ) + monkeypatch.setattr("sys.argv", ["framevitals", "system-info"]) + + assert main() == 0 + rendered = capsys.readouterr().out + assert "FrameVitals system info" in rendered + assert "Backend" in rendered + assert "Native" in rendered + assert "available: True" in rendered diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..1a780bd --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,135 @@ +import pandas as pd +import pytest + +import framevitals +from framevitals.config import AnalysisConfig, available_modules, resolve_config + + +def test_default_and_preset_resolution(): + assert resolve_config() == AnalysisConfig() + + quick = resolve_config(preset="quick") + assert quick.mode == "quick" + assert quick.workers == 2 + assert quick.artifacts is False + assert quick.disabled_modules == () + + ci = resolve_config(preset="ci") + assert ci.mode == "standard" + assert ci.workers == 2 + assert set(ci.disabled_modules) == { + "modeling", + "explainability", + "charts", + "ai", + } + + +def test_toml_config_resolution_and_explicit_precedence(tmp_path): + config = tmp_path / "framevitals.toml" + config.write_text( + "[analysis]\n" + "preset = \"deep\"\n" + "target = \"churn\"\n" + "artifacts = true\n" + "disabled_modules = [\"text_profile\"]\n" + "\n" + "[resources]\n" + "workers = 6\n" + "\n" + "[modules]\n" + "anomaly_detection = false\n", + encoding="utf-8", + ) + + resolved = resolve_config(config) + assert resolved.mode == "deep" + assert resolved.target == "churn" + assert resolved.artifacts is True + assert resolved.workers == 6 + assert resolved.disabled_modules == ("text_profile", "anomaly_detection") + + overridden = resolve_config( + config, + mode="quick", + target="label", + artifacts=False, + workers=3, + disabled_modules=["charts"], + ) + assert overridden == AnalysisConfig( + mode="quick", + target="label", + artifacts=False, + workers=3, + disabled_modules=("charts",), + ) + + +def test_module_booleans_can_selectively_override_preset_defaults(): + resolved = resolve_config( + preset="ci", + config={ + "modules": { + "modeling": True, + "time_series": False, + }, + }, + ) + + assert "modeling" not in resolved.disabled_modules + assert "time_series" in resolved.disabled_modules + assert "explainability" in resolved.disabled_modules + assert "charts" in resolved.disabled_modules + assert "ai" in resolved.disabled_modules + + +def test_config_mapping_and_validation(): + resolved = resolve_config({ + "analysis": {"mode": "research", "artifacts": False}, + "resources": {"workers": 8}, + "modules": {"deep_statistics": False}, + }) + assert resolved.mode == "research" + assert resolved.workers == 8 + assert resolved.disabled_modules == ("deep_statistics",) + + assert "modeling" in available_modules() + assert "cleaning" in available_modules() + + with pytest.raises(ValueError, match="Unknown FrameVitals preset"): + resolve_config(preset="everything") + with pytest.raises(ValueError, match="workers"): + resolve_config(workers=0) + with pytest.raises(ValueError, match="Unknown FrameVitals module"): + resolve_config(disabled_modules=["magic_model"]) + with pytest.raises(ValueError, match="Unknown FrameVitals module in"): + resolve_config({"modules": {"magic_model": False}}) + + +def test_public_analyze_honors_config_and_records_resolution(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + df = pd.DataFrame({ + "age": [20, 30, 40, 50], + "city": ["Pune", "Mumbai", "Pune", "Nashik"], + }) + + result = framevitals.analyze( + df, + config={ + "analysis": {"mode": "quick", "artifacts": False}, + "resources": {"workers": 1}, + }, + ) + + assert result["analysis_mode"] == "quick" + assert result["artifacts_enabled"] is False + assert result["config"] == { + "mode": "quick", + "target": None, + "artifacts": False, + "workers": 1, + "disabled_modules": (), + } + assert result["execution"]["disabled_modules"] == [] + assert not (tmp_path / "cleaned").exists() diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 6d7873e..b925e4c 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -20,55 +20,117 @@ def _reference_dataframe() -> pd.DataFrame: }) -def test_infer_contract_captures_json_safe_schema_and_bounds(): +def test_infer_contract_v2_is_json_safe_and_uses_tolerant_bounds(): contract = infer_contract(_reference_dataframe()) - assert contract == { - "version": 1, - "allow_extra_columns": False, - "columns": { - "account_id": { - "type": "integer", - "nullable": False, - "minimum": 101, - "maximum": 103, - }, - "balance": { - "type": "number", - "nullable": False, - "minimum": 10.5, - "maximum": 40.0, - }, - "plan": {"type": "string", "nullable": False}, - "joined_at": {"type": "datetime", "nullable": False}, - }, - } + assert contract["version"] == 2 + assert contract["allow_extra_columns"] is False + assert contract["extra_columns_severity"] == "error" + assert contract["inference"]["numeric_tolerance"] == 0.05 + + account = contract["columns"]["account_id"] + balance = contract["columns"]["balance"] + plan = contract["columns"]["plan"] + + assert account["minimum"] < 101 + assert account["maximum"] > 103 + assert balance["minimum"] < 10.5 + assert balance["maximum"] > 40.0 + assert plan["allowed_values"] == ["basic", "premium", "standard"] + assert plan["allowed_values_severity"] == "warning" assert json.loads(json.dumps(contract)) == contract -def test_validate_contract_accepts_compatible_data_without_mutating_it(): - contract = infer_contract(_reference_dataframe()) +def test_inferred_contract_accepts_normal_future_values_inside_tolerance(): + contract = infer_contract(_reference_dataframe(), numeric_tolerance=0.10) candidate = pd.DataFrame({ - "account_id": [101, 102], - "balance": [11, 30], + "account_id": [100, 104], + "balance": [9.0, 42.0], "plan": ["basic", "premium"], "joined_at": pd.to_datetime(["2026-01-04", "2026-01-05"]), }) - original = candidate.copy(deep=True) result = validate_contract(candidate, contract) assert result["valid"] is True - assert result["summary"] == { - "columns_checked": 4, - "errors": 0, - "warnings": 0, - } - pd.testing.assert_frame_equal(candidate, original) + assert result["status"] == "pass" + assert result["summary"]["errors"] == 0 -def test_validate_contract_reports_schema_type_nullability_and_bound_errors(): +def test_allowed_value_drift_is_warning_not_failure_by_default(): contract = infer_contract(_reference_dataframe()) + candidate = _reference_dataframe().copy() + candidate.loc[0, "plan"] = "enterprise" + + result = validate_contract(candidate, contract) + + assert result["valid"] is True + assert result["status"] == "warn" + assert result["errors"] == [] + assert result["warnings"][0]["code"] == "allowed_values_violation" + assert result["warnings"][0]["severity"] == "warning" + + +def test_inferred_uniqueness_requires_enough_rows_and_detects_duplicates(): + reference = pd.DataFrame({ + "customer_id": [f"CUST-{index:03d}" for index in range(30)], + "value": list(range(30)), + }) + contract = infer_contract(reference) + + assert contract["columns"]["customer_id"]["unique"] is True + + candidate = reference.copy() + candidate.loc[1, "customer_id"] = candidate.loc[0, "customer_id"] + result = validate_contract(candidate, contract) + + assert result["valid"] is False + assert any(item["code"] == "uniqueness_violation" for item in result["errors"]) + + +def test_nullable_column_can_warn_when_null_rate_exceeds_expected_ceiling(): + contract = { + "version": 2, + "columns": { + "value": { + "type": "number", + "required": True, + "nullable": True, + "max_null_fraction": 0.10, + }, + }, + } + candidate = pd.DataFrame({"value": [1.0, None, None, 4.0]}) + + result = validate_contract(candidate, contract) + + assert result["valid"] is True + assert result["status"] == "warn" + assert result["warnings"][0]["code"] == "null_fraction_violation" + + +def test_optional_contract_column_may_be_absent(): + contract = { + "version": 2, + "columns": { + "required_value": {"type": "integer", "nullable": False}, + "optional_note": { + "type": "string", + "required": False, + "nullable": True, + }, + }, + } + candidate = pd.DataFrame({"required_value": [1, 2, 3]}) + + result = validate_contract(candidate, contract) + + assert result["valid"] is True + assert result["status"] == "pass" + + +def test_validation_aggregates_schema_type_nullability_and_bound_errors(): + contract = infer_contract(_reference_dataframe(), numeric_tolerance=0) candidate = pd.DataFrame({ "balance": [5.0, 55.0], "plan": [1, 2], @@ -77,31 +139,65 @@ def test_validate_contract_reports_schema_type_nullability_and_bound_errors(): }) result = validate_contract(candidate, contract) + codes = {finding["code"] for finding in result["errors"]} assert result["valid"] is False - assert {finding["code"] for finding in result["errors"]} == { + assert result["status"] == "fail" + assert { "missing_column", "unexpected_column", "incompatible_type", "minimum_violation", "maximum_violation", + }.issubset(codes) + assert result["summary"]["findings"] == len(result["findings"]) + assert result["summary"]["failed_columns"] + assert result["summary"]["code_counts"] + + +def test_version_1_contracts_remain_supported(): + contract = { + "version": 1, + "columns": { + "age": { + "type": "integer", + "nullable": False, + "minimum": 18, + "maximum": 100, + }, + }, } + result = validate_contract(pd.DataFrame({"age": [15, 25]}), contract) -def test_validate_contract_rejects_nulls_when_contract_disallows_them(): - contract = infer_contract(_reference_dataframe()) - candidate = _reference_dataframe() - candidate.loc[1, "plan"] = None + assert result["contract_version"] == 1 + assert result["status"] == "fail" + assert any(item["code"] == "minimum_violation" for item in result["errors"]) - result = validate_contract(candidate, contract) - assert result["valid"] is False - assert result["errors"] == [{ - "code": "nullability_violation", - "column": "plan", - "message": "Column 'plan' does not allow null values (1 found).", - "null_count": 1, - }] +def test_extra_columns_can_be_allowed_or_downgraded_to_warning(): + candidate = pd.DataFrame({"age": [20], "new_col": [1]}) + + allowed = validate_contract( + candidate, + { + "version": 2, + "allow_extra_columns": True, + "columns": {"age": {"type": "integer", "nullable": False}}, + }, + ) + warning = validate_contract( + candidate, + { + "version": 2, + "extra_columns_severity": "warning", + "columns": {"age": {"type": "integer", "nullable": False}}, + }, + ) + + assert allowed["status"] == "pass" + assert warning["status"] == "warn" + assert warning["warnings"][0]["code"] == "unexpected_column" def test_validate_contract_rejects_invalid_contract_definitions(): @@ -122,16 +218,37 @@ def test_validate_contract_rejects_invalid_contract_definitions(): }, ) + with pytest.raises(ValueError, match="between 0 and 1"): + validate_contract( + _reference_dataframe(), + { + "version": 2, + "columns": { + "balance": { + "type": "number", + "max_null_fraction": 1.5, + }, + }, + }, + ) + -def test_public_contract_api_accepts_file_paths_and_preserves_source_name(tmp_path): +def test_public_contract_api_accepts_file_paths_and_inference_controls(tmp_path): reference_path = tmp_path / "reference.csv" candidate_path = tmp_path / "candidate.csv" _reference_dataframe().to_csv(reference_path, index=False) _reference_dataframe().to_csv(candidate_path, index=False) - contract = framevitals.infer_contract(reference_path) + contract = framevitals.infer_contract( + reference_path, + numeric_tolerance=0.10, + max_categories=10, + allow_extra_columns=True, + ) result = framevitals.validate(candidate_path, contract) assert contract["reference_name"] == "reference.csv" + assert contract["inference"]["numeric_tolerance"] == 0.10 + assert contract["allow_extra_columns"] is True assert result["dataset_name"] == "candidate.csv" assert result["valid"] is True diff --git a/tests/test_csv_streaming.py b/tests/test_csv_streaming.py new file mode 100644 index 0000000..1918641 --- /dev/null +++ b/tests/test_csv_streaming.py @@ -0,0 +1,96 @@ +import numpy as np +import pandas as pd +import pytest + +pytest.importorskip("pyarrow") + +import framevitals +from framevitals.sources import DelimitedTextSource, resolve_source + + +def _frame(rows: int) -> pd.DataFrame: + frame = pd.DataFrame({ + "value": np.arange(rows, dtype=np.float64), + "other": np.arange(rows, dtype=np.float64) * 1.5, + "group": [f"g-{index % 6}" for index in range(rows)], + }) + frame.loc[::137, "value"] = np.nan + return frame + + +def test_csv_source_exposes_exact_streaming_metadata_and_projection(tmp_path): + path = tmp_path / "dataset.csv" + frame = _frame(4_000) + frame.to_csv(path, index=False) + + source = resolve_source(path) + assert isinstance(source, DelimitedTextSource) + + metadata = source.inspect() + assert metadata.format == "csv" + assert metadata.rows == len(frame) + assert metadata.columns == len(frame.columns) + assert metadata.materialized is False + assert metadata.supports_projection is True + assert metadata.supports_streaming is True + + batches = list(source.iter_batches(batch_size=700, columns=["value", "group"])) + assert sum(batch.num_rows for batch in batches) == len(frame) + assert max(batch.num_rows for batch in batches) <= 700 + assert batches[0].schema.names == ["value", "group"] + + +def test_public_profile_streams_csv_without_calling_load(tmp_path, monkeypatch): + path = tmp_path / "stream.csv" + frame = _frame(12_000) + frame.to_csv(path, index=False) + + def fail_load(self): + raise AssertionError("streaming CSV profile must not materialize the complete file") + + monkeypatch.setattr(DelimitedTextSource, "load", fail_load) + result = framevitals.profile(path) + + assert result["dataset_name"] == "stream.csv" + assert result["shape"] == {"rows": len(frame), "columns": len(frame.columns)} + assert result["streaming_metadata"]["enabled"] is True + assert result["streaming_metadata"]["full_materialization"] is False + assert result["source_metadata"]["format"] == "csv" + assert result["source_metadata"]["supports_streaming"] is True + assert result["missing_counts"]["value"] == int(frame["value"].isna().sum()) + + +def test_public_analyze_dispatches_csv_through_streaming_pipeline(tmp_path, monkeypatch): + path = tmp_path / "analyze.csv" + frame = _frame(6_000) + frame.to_csv(path, index=False) + + def fail_load(self): + raise AssertionError("streaming CSV analysis must not materialize the complete file") + + monkeypatch.setattr(DelimitedTextSource, "load", fail_load) + result = framevitals.analyze(path, mode="quick", artifacts=False, workers=1) + + assert result["filename"] == "analyze.csv" + assert result["profile"]["shape"] == {"rows": len(frame), "columns": len(frame.columns)} + assert result["execution"]["streaming"]["enabled"] is True + assert result["execution"]["streaming"]["full_materialization"] is False + assert result["execution"]["streaming"]["source_rows"] == len(frame) + + +def test_tsv_source_streams_with_tab_delimiter(tmp_path): + path = tmp_path / "dataset.tsv" + frame = _frame(1_500) + frame.to_csv(path, index=False, sep="\t") + + source = resolve_source(path) + assert isinstance(source, DelimitedTextSource) + metadata = source.inspect() + + assert metadata.format == "tsv" + assert metadata.rows == len(frame) + assert metadata.columns == len(frame.columns) + assert metadata.supports_streaming is True + batch = next(source.iter_batches(batch_size=200, columns=["group"])) + assert batch.schema.names == ["group"] + assert batch.num_rows <= 200 diff --git a/tests/test_custom_checks.py b/tests/test_custom_checks.py new file mode 100644 index 0000000..10d8f41 --- /dev/null +++ b/tests/test_custom_checks.py @@ -0,0 +1,176 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pandas as pd +import pytest + +import framevitals as fv + + +def test_top_level_import_keeps_custom_check_engine_lazy(): + src_root = Path(__file__).resolve().parents[1] / "src" + env = os.environ.copy() + env["PYTHONPATH"] = str(src_root) + completed = subprocess.run( + [ + sys.executable, + "-c", + "import sys; import framevitals; assert 'framevitals.checks' not in sys.modules", + ], + cwd=src_root.parent, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + + +def test_check_decorator_and_exact_check_results(): + frame = pd.DataFrame({"revenue": [10.0, 20.0, 30.0]}) + + @fv.check("positive revenue", description="Revenue cannot be negative.") + def positive_revenue(df): + passed = bool((df["revenue"] >= 0).all()) + return { + "passed": passed, + "message": "Revenue is non-negative." if passed else "Negative revenue found.", + "details": {"minimum": float(df["revenue"].min())}, + } + + assert positive_revenue.name == "positive revenue" + result = fv.run_checks(frame, [positive_revenue]) + + assert isinstance(result, fv.CheckResult) + assert result.status == "pass" + assert result.passed is True + assert result["summary"] == {"checks": 1, "passed": 1, "warnings": 0, "errors": 0} + assert result.results[0]["code"] == "custom.positive_revenue" + assert result.results[0]["details"] == {"minimum": 10.0} + assert result.findings == [] + assert "FrameVitals custom checks" in result.summary_text() + assert result["execution"]["method"] == "exact_custom_checks" + assert result["execution"]["full_materialization"] is False + + +def test_warning_and_error_checks_produce_distinct_statuses(): + frame = pd.DataFrame({"value": [1, 2, 3]}) + + @fv.check("soft expectation", severity="warning") + def soft_expectation(df): + return False + + @fv.check("hard expectation", severity="error") + def hard_expectation(df): + return {"passed": False, "message": "Hard invariant failed."} + + warning_only = fv.run_checks(frame, [soft_expectation]) + assert warning_only["status"] == "warn" + assert warning_only["passed"] is True + assert warning_only["summary"]["warnings"] == 1 + + with_error = fv.run_checks(frame, [soft_expectation, hard_expectation]) + assert with_error["status"] == "fail" + assert with_error["passed"] is False + assert with_error["summary"]["warnings"] == 1 + assert with_error["summary"]["errors"] == 1 + assert {finding["code"] for finding in with_error["findings"]} == { + "custom.soft_expectation", + "custom.hard_expectation", + } + + +def test_custom_check_exceptions_become_structured_failures(): + frame = pd.DataFrame({"value": [1, 2, 3]}) + + @fv.check("exploding rule") + def exploding_rule(df): + raise RuntimeError("boom") + + result = fv.run_checks(frame, [exploding_rule]) + + assert result["status"] == "fail" + check_result = result["results"][0] + assert check_result["passed"] is False + assert check_result["severity"] == "error" + assert "RuntimeError: boom" in check_result["execution_error"] + assert "RuntimeError" in check_result["message"] + + +def test_each_custom_check_receives_an_isolated_dataframe_copy(): + frame = pd.DataFrame({"value": [1, 2, 3]}) + + @fv.check("mutating check") + def mutating_check(df): + df["temporary"] = 1 + return True + + @fv.check("isolation check") + def isolation_check(df): + return "temporary" not in df.columns + + result = fv.run_checks(frame, [mutating_check, isolation_check]) + + assert result["status"] == "pass" + assert "temporary" not in frame.columns + + +def test_file_backed_custom_checks_disclose_full_materialization(tmp_path): + path = tmp_path / "dataset.csv" + pd.DataFrame({"value": [1, 2, 3]}).to_csv(path, index=False) + + result = fv.run_checks(path, [lambda df: bool((df["value"] > 0).all())]) + + assert result["status"] == "pass" + assert result["execution"]["full_materialization"] is True + assert result["execution"]["source"]["kind"] == "file" + + +def test_gate_can_run_only_custom_warning_checks(): + frame = pd.DataFrame({"latency_ms": [10, 15, 20]}) + + @fv.check("latency budget", severity="warning") + def latency_budget(df): + return { + "passed": bool(df["latency_ms"].max() < 15), + "message": "Latency exceeded the preferred budget.", + } + + result = fv.gate(frame, custom_checks=[latency_budget]) + + assert result["status"] == "warn" + assert result["passed"] is True + assert result["checks_run"] == ["custom"] + assert result["checks"]["custom"]["status"] == "warn" + assert result["execution"]["full_materialization"] is False + assert result["execution"]["custom"]["method"] == "exact_custom_checks" + assert result["execution"]["drift"] is None + assert result["execution"]["validation"] is None + assert "Latency exceeded the preferred budget." in result["reasons"] + + +def test_gate_custom_error_fails_quality_gate(): + frame = pd.DataFrame({"revenue": [100, -5, 80]}) + + @fv.check("positive revenue") + def positive_revenue(df): + return { + "passed": bool((df["revenue"] >= 0).all()), + "message": "Negative revenue records are not allowed.", + } + + result = fv.gate(frame, custom_checks=[positive_revenue]) + + assert result["status"] == "fail" + assert result["passed"] is False + assert result["checks"]["custom"]["summary"]["errors"] == 1 + assert "Negative revenue records are not allowed." in result["reasons"] + + +def test_gate_still_requires_at_least_one_check_family(): + frame = pd.DataFrame({"value": [1]}) + + with pytest.raises(ValueError, match="custom_checks"): + fv.gate(frame) diff --git a/tests/test_deep_triage.py b/tests/test_deep_triage.py new file mode 100644 index 0000000..83b9642 --- /dev/null +++ b/tests/test_deep_triage.py @@ -0,0 +1,70 @@ +import numpy as np +import pandas as pd + +from framevitals.budgeted_analysis import run_budgeted_deep_statistics +from framevitals.deep_triage import triage_deep_columns +from framevitals.execution import derive_execution_budget + + +def _wide_frame(rows: int = 2_000) -> pd.DataFrame: + rng = np.random.default_rng(42) + payload: dict[str, object] = {} + for index in range(30): + payload[f"n{index}"] = rng.normal(size=rows) + payload["n27"] = rng.lognormal(mean=0.0, sigma=2.0, size=rows) + payload["n28"] = np.where(np.arange(rows) % 4 == 0, np.nan, rng.normal(size=rows)) + payload["n29"] = np.ones(rows) + for index in range(15): + payload[f"c{index}"] = np.array([f"g-{value % (index + 2)}" for value in range(rows)], dtype=object) + payload["c14"] = np.where(np.arange(rows) % 3 == 0, None, "rare") + return pd.DataFrame(payload) + + +def test_deep_triage_bounds_columns_and_prioritizes_diagnostic_signals(): + frame = _wide_frame() + result = triage_deep_columns(frame, mode="deep") + + assert len(result.selected_numeric) == 12 + assert len(result.selected_categorical) == 8 + assert {"n27", "n28", "n29"} <= set(result.selected_numeric) + assert "c14" in result.selected_categorical + metadata = result.to_dict() + assert metadata["numeric_truncated"] is True + assert metadata["categorical_truncated"] is True + + +def test_research_mode_keeps_a_larger_deep_diagnostic_surface(): + frame = _wide_frame() + deep = triage_deep_columns(frame, mode="deep") + research = triage_deep_columns(frame, mode="research") + + assert len(research.selected_numeric) > len(deep.selected_numeric) + assert len(research.selected_categorical) > len(deep.selected_categorical) + assert set(deep.selected_numeric) <= set(research.selected_numeric) + + +def test_budgeted_deep_statistics_only_passes_triaged_columns(monkeypatch): + frame = _wide_frame(rows=4_000) + seen: dict[str, object] = {} + + def fake_deep(diagnostic_view, max_pairs=20): + seen["columns"] = list(diagnostic_view.columns) + seen["rows"] = len(diagnostic_view) + seen["pairs"] = max_pairs + return {"available": True} + + monkeypatch.setattr( + "framevitals.budgeted_analysis.run_fast_deep_statistics_v2", + fake_deep, + ) + budget = derive_execution_budget(len(frame), len(frame.columns), mode="deep") + result = run_budgeted_deep_statistics(frame, budget=budget) + + assert len(seen["columns"]) <= 20 + assert seen["rows"] <= budget.bootstrap_sample_rows + assert result["column_triage"]["numeric_limit"] == 12 + assert result["column_triage"]["categorical_limit"] == 8 + assert result["execution"]["method"] == "bounded_deep_statistics" + assert result["execution"]["scope"] == "bounded_deep_statistics" + assert result["execution"]["adaptive_strategy"] == "column_interest_triage" + assert result["execution"]["diagnostic_columns"] == len(seen["columns"]) diff --git a/tests/test_diagnostic_results.py b/tests/test_diagnostic_results.py new file mode 100644 index 0000000..08bb938 --- /dev/null +++ b/tests/test_diagnostic_results.py @@ -0,0 +1,90 @@ +import json + +import numpy as np +import pandas as pd + +import framevitals as fv + + +def _frame(rows: int = 120) -> pd.DataFrame: + values = np.arange(rows, dtype=np.float64) + return pd.DataFrame({ + "value": values, + "other": values * 2.0 + 1.0, + "group": [f"g-{index % 4}" for index in range(rows)], + "target": [index % 2 for index in range(rows)], + }) + + +def test_profile_returns_dict_compatible_diagnostic_result_without_schema_mutation(tmp_path): + result = fv.profile(_frame()) + + assert isinstance(result, dict) + assert isinstance(result, fv.DiagnosticResult) + assert result.diagnostic == "profile" + assert result.dataset_name == "" + assert result.available is True + assert "diagnostic" not in result + + detached = result.to_dict() + assert type(detached) is dict + assert detached == dict(result) + detached["shape"]["rows"] = -1 + assert result["shape"]["rows"] == 120 + + rendered = json.loads(result.to_json()) + assert rendered == dict(result) + assert "diagnostic" not in rendered + + destination = tmp_path / "profile.json" + returned = result.to_json(destination) + assert returned == destination + assert json.loads(destination.read_text(encoding="utf-8")) == dict(result) + + +def test_focused_apis_label_their_diagnostic_result(): + frame = _frame() + + results = { + "roles": fv.roles(frame), + "health": fv.health(frame), + "ml_readiness": fv.ml_readiness(frame), + "quality": fv.quality(frame), + "statistics": fv.statistics(frame, mode="quick", max_pairs=2), + "anomalies": fv.anomalies(frame, mode="quick", max_columns=3, top_k=5), + "relationships": fv.relationships(frame, max_sample_rows=64), + "target_analysis": fv.target_analysis(frame, target="target"), + } + + for diagnostic, result in results.items(): + assert isinstance(result, fv.DiagnosticResult) + assert result.diagnostic == diagnostic + assert result.dataset_name == "" + assert "diagnostic" not in result + + +def test_diagnostic_result_execution_and_source_helpers_follow_public_provenance(): + result = fv.statistics(_frame(800), mode="quick", max_pairs=2) + + assert result.execution["execution_schema_version"] == "1" + assert result.execution["method"] == "bounded_deep_statistics" + assert result.source["format"] == "pandas" + assert result.source["rows"] == 800 + + summary = result.summary() + assert summary["diagnostic"] == "statistics" + assert summary["dataset_name"] == "" + assert summary["method"] == "bounded_deep_statistics" + assert summary["execution_schema_version"] == "1" + assert summary["source_rows"] == 800 + assert "FrameVitals statistics" in result.summary_text() + + +def test_unavailable_diagnostic_result_exposes_available_false(): + frame = pd.DataFrame({"label": ["a", "b", "c", "d"]}) + result = fv.anomalies(frame, mode="quick") + + assert isinstance(result, fv.DiagnosticResult) + assert result.available is False + assert result["available"] is False + assert result.execution["method"] == "bounded_anomaly_detection" diff --git a/tests/test_duckdb_source.py b/tests/test_duckdb_source.py new file mode 100644 index 0000000..dc6909e --- /dev/null +++ b/tests/test_duckdb_source.py @@ -0,0 +1,175 @@ +import pandas as pd +import pytest + +pytest.importorskip("pyarrow") +duckdb = pytest.importorskip("duckdb") + +import framevitals +from framevitals.duckdb_source import DuckDBRelationSource +from framevitals.sources import resolve_source + + +def _relation(rows: int = 12_000): + connection = duckdb.connect() + relation = connection.sql( + f""" + SELECT + range::DOUBLE AS value, + (range * 2)::DOUBLE AS other, + concat('g-', range % 5) AS grp + FROM range({int(rows)}) + """ + ) + return connection, relation + + +def _frame(rows: int): + return pd.DataFrame({ + "value": [float(index) for index in range(rows)], + "other": [float(index * 2) for index in range(rows)], + "grp": [f"g-{index % 5}" for index in range(rows)], + }) + + +def test_duckdb_relation_source_exposes_exact_metadata_projection_and_batches(): + connection, relation = _relation(4_000) + try: + source = resolve_source(relation) + assert isinstance(source, DuckDBRelationSource) + + metadata = source.inspect() + assert metadata.name == "" + assert metadata.kind == "relation" + assert metadata.format == "duckdb" + assert metadata.rows == 4_000 + assert metadata.columns == 3 + assert metadata.size_bytes is None + assert metadata.materialized is False + assert metadata.supports_projection is True + assert metadata.supports_streaming is True + assert source.schema().names == ["value", "other", "grp"] + + public_info = framevitals.inspect_source(relation) + assert public_info == metadata.to_dict() + + batches = list(source.iter_batches(batch_size=700, columns=["value", "grp"])) + assert sum(batch.num_rows for batch in batches) == 4_000 + assert max(batch.num_rows for batch in batches) <= 700 + assert batches[0].schema.names == ["value", "grp"] + finally: + connection.close() + + +def test_public_profile_streams_duckdb_relation_without_pandas_materialization(monkeypatch): + connection, relation = _relation(12_000) + + def fail_load(self): + raise AssertionError("DuckDB profile must not materialize the complete relation") + + monkeypatch.setattr(DuckDBRelationSource, "load", fail_load) + try: + result = framevitals.profile(relation) + finally: + connection.close() + + assert result["dataset_name"] == "" + assert result["shape"] == {"rows": 12_000, "columns": 3} + assert result["streaming_metadata"]["enabled"] is True + assert result["streaming_metadata"]["full_materialization"] is False + assert result["source_metadata"]["kind"] == "relation" + assert result["source_metadata"]["format"] == "duckdb" + + +def test_public_analyze_streams_duckdb_relation(monkeypatch): + connection, relation = _relation(6_000) + + def fail_load(self): + raise AssertionError("DuckDB analysis must stay on the streaming relation path") + + monkeypatch.setattr(DuckDBRelationSource, "load", fail_load) + try: + result = framevitals.analyze( + relation, + mode="quick", + artifacts=False, + workers=1, + ) + finally: + connection.close() + + assert result["filename"] == "" + assert result["profile"]["shape"] == {"rows": 6_000, "columns": 3} + assert result["execution"]["streaming"]["enabled"] is True + assert result["execution"]["streaming"]["full_materialization"] is False + assert result["execution"]["streaming"]["source_rows"] == 6_000 + + +def test_plan_uses_bounded_duckdb_relation_sample(monkeypatch): + connection, relation = _relation(20_000) + + def fail_load(self): + raise AssertionError("DuckDB planning must not load the complete relation") + + monkeypatch.setattr(DuckDBRelationSource, "load", fail_load) + try: + plan = framevitals.plan(relation, mode="standard") + finally: + connection.close() + + assert plan["dataset_name"] == "" + assert plan["source"]["format"] == "duckdb" + assert plan["shape"] == {"rows": 20_000, "columns": 3} + assert plan["planning_data"]["materialized_full_dataset"] is False + assert plan["planning_data"]["sampled"] is True + assert plan["planning_data"]["sample_rows"] == 5_000 + + +def test_exact_contract_validation_reports_duckdb_materialization(): + rows = 100 + contract = framevitals.infer_contract(_frame(rows)) + connection, relation = _relation(rows) + try: + result = framevitals.validate(relation, contract) + finally: + connection.close() + + assert result["status"] in {"pass", "warn"} + assert result["execution"]["method"] == "exact_contract_validation" + assert result["execution"]["full_materialization"] is True + assert result["execution"]["source"]["format"] == "duckdb" + + +def test_exact_custom_checks_report_duckdb_materialization(): + connection, relation = _relation(100) + try: + result = framevitals.run_checks( + relation, + [lambda df: bool((df["value"] >= 0).all())], + ) + finally: + connection.close() + + assert isinstance(result, framevitals.CheckResult) + assert result.status == "pass" + assert result["execution"]["full_materialization"] is True + assert result["execution"]["source"]["format"] == "duckdb" + + +def test_drift_only_gate_keeps_duckdb_relations_streaming(monkeypatch): + reference_connection, reference = _relation(1_000) + current_connection, current = _relation(1_000) + + def fail_load(self): + raise AssertionError("Drift-only DuckDB gate must stay on the streaming path") + + monkeypatch.setattr(DuckDBRelationSource, "load", fail_load) + try: + result = framevitals.gate(current, reference=reference) + finally: + reference_connection.close() + current_connection.close() + + assert result.status in {"pass", "warn"} + assert result["checks_run"] == ["drift"] + assert result["execution"]["full_materialization"] is False + assert result["execution"]["drift"]["full_materialization"] is False diff --git a/tests/test_execution_controls.py b/tests/test_execution_controls.py new file mode 100644 index 0000000..a76685c --- /dev/null +++ b/tests/test_execution_controls.py @@ -0,0 +1,141 @@ +import pandas as pd + +import framevitals +from framevitals.cli import build_parser, main + + +def _frame(rows: int = 36) -> pd.DataFrame: + return pd.DataFrame({ + "value": list(range(rows)), + "value_2": [index * 2 for index in range(rows)], + "segment": ["A", "B", "C"] * (rows // 3), + "target": [0, 1] * (rows // 2), + }) + + +def test_analyze_can_disable_expensive_modules_without_losing_result_keys(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + result = framevitals.analyze( + _frame(), + target="target", + mode="standard", + artifacts=True, + disabled_modules=[ + "quality_diagnostics", + "deep_statistics", + "anomaly_detection", + "time_series", + "text_profile", + "target_intelligence", + "modeling", + "explainability", + "cleaning", + "charts", + "ai", + ], + ) + + assert result["execution"]["disabled_modules"] == sorted( + framevitals.available_modules() + ) + assert all( + status == "disabled" + for status in result["execution"]["module_status"].values() + ) + + assert result["quality_diagnostics"]["skipped"] is True + assert result["deep_statistics_v2"]["skipped"] is True + assert result["anomalies_v2"]["skipped"] is True + assert result["time_series"]["skipped"] is True + assert result["text_profile"]["skipped"] is True + assert result["target_intelligence"]["skipped"] is True + assert result["model_leaderboard"]["skipped"] is True + assert result["explainability"]["skipped"] is True + assert result["cleaning"]["skipped"] is True + assert result["charts"] == [] + assert result["ai_report"]["source"] == "disabled" + + assert not (tmp_path / "cleaned").exists() + assert not (tmp_path / "static" / "charts").exists() + + +def test_ci_preset_skips_modeling_but_keeps_target_intelligence_and_quality(): + result = framevitals.analyze( + _frame(), + target="target", + preset="ci", + ) + + execution = result["execution"]["module_status"] + assert execution["modeling"] == "disabled" + assert execution["explainability"] == "disabled" + assert execution["charts"] == "disabled" + assert execution["ai"] == "disabled" + assert execution["target_intelligence"] == "ran" + assert execution["quality_diagnostics"] == "ran" + + assert result["model_leaderboard"]["skipped"] is True + assert result["target_intelligence"]["available"] is True + assert result["quality_diagnostics"]["available"] is True + + +def test_plan_surfaces_execution_module_selection(): + plan = framevitals.plan( + _frame(), + target="target", + mode="deep", + disabled_modules=["modeling", "charts"], + ) + + modules = plan["selection"]["execution_modules"] + assert modules["disabled"] == ["charts", "modeling"] + assert "anomaly_detection" in modules["enabled"] + assert "quality_diagnostics" in modules["enabled"] + assert plan["config"]["disabled_modules"] == ("modeling", "charts") + + +def test_cli_accepts_repeatable_disable_module_flags(): + parser = build_parser() + args = parser.parse_args([ + "analyze", + "data.csv", + "--disable-module", + "modeling", + "--disable-module", + "charts", + ]) + + assert args.disabled_modules == ["modeling", "charts"] + + +def test_cli_module_flags_reach_pipeline(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "data.csv" + _frame().to_csv(dataset, index=False) + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "analyze", + str(dataset), + "--mode", + "standard", + "--disable-module", + "anomaly_detection", + "--disable-module", + "modeling", + "--format", + "json", + ], + ) + + assert main() == 0 + payload = __import__("json").loads(capsys.readouterr().out) + assert payload["execution"]["module_status"]["anomaly_detection"] == "disabled" + assert payload["anomalies_v2"]["skipped"] is True + + # Modeling requires a target in the first place. The configuration still + # records it as disabled, but execution correctly explains that this run + # skipped it because it was not applicable. + assert "modeling" in payload["execution"]["disabled_modules"] + assert payload["execution"]["module_status"]["modeling"] == "not_applicable" diff --git a/tests/test_extreme_streaming_projection.py b/tests/test_extreme_streaming_projection.py new file mode 100644 index 0000000..f480390 --- /dev/null +++ b/tests/test_extreme_streaming_projection.py @@ -0,0 +1,106 @@ +from types import SimpleNamespace + +import numpy as np +import pytest + +pa = pytest.importorskip("pyarrow") + +import framevitals as fv +from framevitals.execution import derive_streaming_profile_column_limit +from framevitals.sources import DatasetMetadata + + +class _VirtualSchema: + def __init__(self, columns: int): + self.columns = int(columns) + self.dtype = pa.float64() + + def __iter__(self): + for index in range(self.columns): + yield SimpleNamespace(name=f"n{index:05d}", type=self.dtype) + + def field(self, name: str): + index = int(name[1:]) + if index < 0 or index >= self.columns: + raise KeyError(name) + return SimpleNamespace(name=name, type=self.dtype) + + +class _VirtualWideSource: + def __init__(self, *, rows: int, columns: int): + self.rows = int(rows) + self.columns = int(columns) + self.max_requested_columns = 0 + self.unbounded_requested = False + self._schema = _VirtualSchema(columns) + + def inspect(self): + return DatasetMetadata( + name="virtual-wide", + kind="virtual", + format="synthetic", + rows=self.rows, + columns=self.columns, + size_bytes=self.rows * self.columns * 8, + materialized=False, + supports_projection=True, + supports_streaming=True, + ) + + def schema(self): + return self._schema + + def iter_batches(self, *, batch_size=65_536, columns=None): + if columns is None: + self.unbounded_requested = True + raise AssertionError("ultra-wide source must always be projected") + names = list(columns) + self.max_requested_columns = max(self.max_requested_columns, len(names)) + rows = min(int(batch_size), self.rows) + values = pa.array(np.arange(rows, dtype=np.float64)) + yield pa.RecordBatch.from_arrays([values] * len(names), names=names) + + def load(self): + raise AssertionError("virtual wide source must never materialize") + + +def test_extreme_deep_streaming_profile_column_limit_is_128(): + assert ( + derive_streaming_profile_column_limit( + 1_000_000, + 100_000, + mode="deep", + ) + == 128 + ) + + +def test_extreme_plan_projects_100k_column_schema(): + source = _VirtualWideSource(rows=1_000_000, columns=100_000) + + result = fv.plan(source, mode="deep") + + assert result["shape"] == {"rows": 1_000_000, "columns": 100_000} + assert result["execution_budget"]["scale_class"] == "extreme" + assert result["planning_data"]["column_sampled"] is True + assert result["planning_data"]["sample_columns"] == 128 + assert result["planning_data"]["source_columns"] == 100_000 + assert source.unbounded_requested is False + assert source.max_requested_columns == 128 + + +def test_ultra_wide_streaming_analysis_never_requests_full_width(): + source = _VirtualWideSource(rows=1_000, columns=10_000) + + result = fv.analyze(source, mode="deep", artifacts=False, workers=2) + + streaming = result["execution"]["streaming"] + assert streaming["source_rows"] == 1_000 + assert streaming["source_columns"] == 10_000 + assert streaming["profiled_columns"] == 128 + assert streaming["column_sampled"] is True + assert streaming["column_strategy"] == "deterministic_schema_projection" + assert result["profile"]["shape"] == {"rows": 1_000, "columns": 10_000} + assert result["health"]["execution"]["profiled_columns"] == 128 + assert source.unbounded_requested is False + assert source.max_requested_columns == 128 diff --git a/tests/test_fast_anomaly.py b/tests/test_fast_anomaly.py new file mode 100644 index 0000000..36e3604 --- /dev/null +++ b/tests/test_fast_anomaly.py @@ -0,0 +1,37 @@ +import numpy as np +import pandas as pd + +from framevitals.fast_anomaly import fast_anomaly_scan + + +def test_fast_anomaly_scan_detects_injected_multivariate_shift(): + rng = np.random.default_rng(42) + rows = 2_000 + matrix = rng.normal(size=(rows, 8)) + matrix[-12:] += 9.0 + frame = pd.DataFrame(matrix, columns=[f"x{i}" for i in range(8)]) + + result = fast_anomaly_scan(frame, contamination=0.02, top_k=25) + + assert result["available"] is True + assert result["method"] == "fast_robust_random_projection" + assert result["detectors_run"] == [ + "robust_feature_deviation", + "random_projection_tail", + "random_projection_density", + ] + assert result["n_rows_scored"] == rows + top_indices = {int(item["row_index"]) for item in result["top_rows"][:20]} + assert len(top_indices & set(range(rows - 12, rows))) >= 8 + + +def test_fast_anomaly_scan_bounds_numeric_dimensions(): + rng = np.random.default_rng(7) + frame = pd.DataFrame(rng.normal(size=(500, 80))) + + result = fast_anomaly_scan(frame, max_columns=16, projections=8) + + assert result["available"] is True + assert len(result["used_columns"]) == 16 + assert result["preparation"]["truncated_columns"] is True + assert result["projection_count"] == 8 diff --git a/tests/test_fast_deep_statistics.py b/tests/test_fast_deep_statistics.py new file mode 100644 index 0000000..c4d8553 --- /dev/null +++ b/tests/test_fast_deep_statistics.py @@ -0,0 +1,58 @@ +import numpy as np +import pandas as pd + +from framevitals.fast_deep_statistics import ( + fast_mean_ci, + fast_median_ci, + run_fast_deep_statistics_v2, +) + + +def test_fast_mean_ci_matches_normal_sample_location(): + rng = np.random.default_rng(42) + series = pd.Series(rng.normal(loc=3.0, scale=1.5, size=2_000)) + + result = fast_mean_ci(series) + + assert result["available"] is True + assert result["method"] == "student_t" + assert result["n_resamples"] == 0 + assert result["low"] < series.mean() < result["high"] + + +def test_fast_median_ci_is_distribution_free_and_contains_median(): + rng = np.random.default_rng(7) + series = pd.Series(rng.lognormal(mean=0.3, sigma=1.1, size=2_000)) + + result = fast_median_ci(series) + + assert result["available"] is True + assert result["method"] == "distribution_free_order_statistic" + assert result["n_resamples"] == 0 + assert result["rank_low"] < result["rank_high"] + assert result["low"] <= series.median() <= result["high"] + + +def test_fast_deep_statistics_preserves_v2_shape(): + rng = np.random.default_rng(11) + frame = pd.DataFrame({ + "x": rng.normal(size=120), + "y": rng.gamma(shape=2.0, scale=1.0, size=120), + "group": np.where(np.arange(120) % 2, "A", "B"), + }) + + result = run_fast_deep_statistics_v2(frame, max_pairs=5) + + assert result["version"] == "v2" + assert result["numeric_columns"] == ["x", "y"] + assert result["categorical_columns"] == ["group"] + assert result["numeric_statistics"]["x"]["bootstrap_mean_ci"]["n_resamples"] == 0 + assert result["numeric_statistics"]["x"]["bootstrap_median_ci"]["n_resamples"] == 0 + assert result["inference"]["bootstrap_resamples"] == 0 + + +def test_fast_ci_rejects_tiny_samples_like_existing_bootstrap_contract(): + series = pd.Series(np.arange(10, dtype=float)) + + assert fast_mean_ci(series) == {"available": False, "reason": "n<20"} + assert fast_median_ci(series) == {"available": False, "reason": "n<20"} diff --git a/tests/test_focused_api.py b/tests/test_focused_api.py new file mode 100644 index 0000000..97a5efa --- /dev/null +++ b/tests/test_focused_api.py @@ -0,0 +1,123 @@ +from pathlib import Path + +import pandas as pd +import pytest + +import framevitals as fv +from framevitals.health_score import calculate_health_score +from framevitals.profiler import build_profile + + +def _frame(rows: int = 40) -> pd.DataFrame: + return pd.DataFrame({ + "age": list(range(20, 20 + rows)), + "income": [30_000 + index * 1_000 for index in range(rows)], + "city": ["Pune", "Mumbai"] * (rows // 2), + "target": [0, 1] * (rows // 2), + }) + + +def test_profile_is_focused_and_matches_profiler(): + df = _frame() + + result = fv.profile(df) + expected = build_profile(df) + + assert result["dataset_name"] == "" + assert result["shape"] == expected["shape"] + assert result["dtypes"] == expected["dtypes"] + assert result["missing_counts"] == expected["missing_counts"] + + +def test_roles_health_and_ml_readiness_are_public(): + df = _frame() + + role_result = fv.roles(df) + health_result = fv.health(df) + readiness_result = fv.ml_readiness(df) + + assert role_result["dataset_name"] == "" + assert set(role_result["columns"]) == set(df.columns) + assert "summary" in role_result + + expected_health = calculate_health_score(df, build_profile(df)) + assert health_result["overall_score"] == expected_health["overall_score"] + assert health_result["label"] == expected_health["label"] + + assert readiness_result["dataset_name"] == "" + assert 0 <= readiness_result["score"] <= 100 + + +def test_quality_statistics_and_anomalies_can_run_independently(): + df = _frame() + + quality = fv.quality(df, max_sample_rows=20) + statistics = fv.statistics(df, max_pairs=5) + anomalies = fv.anomalies(df, top_k=5) + + assert quality["dataset_name"] == "" + assert quality["available"] is True + assert quality["max_sample_rows"] == 20 + + assert statistics["dataset_name"] == "" + assert isinstance(statistics, dict) + + assert anomalies["dataset_name"] == "" + assert anomalies["available"] is True + assert len(anomalies["top_rows"]) <= 5 + + +def test_target_analysis_is_focused_and_validates_target(): + df = _frame() + + result = fv.target_analysis(df, target="target") + + assert result["dataset_name"] == "" + assert result["available"] is True + assert result["target_column"] == "target" + assert result["task_type"] == "classification" + + with pytest.raises(ValueError, match="Target column not found"): + fv.target_analysis(df, target="does_not_exist") + + +def test_focused_apis_preserve_file_source_name(tmp_path): + path = tmp_path / "customers.csv" + _frame().to_csv(path, index=False) + + assert fv.profile(path)["dataset_name"] == "customers.csv" + assert fv.quality(path)["dataset_name"] == "customers.csv" + assert fv.anomalies(path)["dataset_name"] == "customers.csv" + + +def test_focused_apis_are_side_effect_free(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + df = _frame() + + fv.profile(df) + fv.roles(df) + fv.health(df) + fv.ml_readiness(df) + fv.quality(df) + fv.statistics(df, max_pairs=5) + fv.anomalies(df, top_k=5) + fv.target_analysis(df, target="target") + + assert not Path("cleaned").exists() + assert not Path("static/charts").exists() + assert not Path("reports").exists() + + +def test_focused_functions_are_exposed_from_top_level(): + for name in ( + "profile", + "roles", + "health", + "ml_readiness", + "quality", + "statistics", + "anomalies", + "target_analysis", + ): + assert callable(getattr(fv, name)) + assert name in fv.__all__ diff --git a/tests/test_focused_import_boundaries.py b/tests/test_focused_import_boundaries.py new file mode 100644 index 0000000..9f3f086 --- /dev/null +++ b/tests/test_focused_import_boundaries.py @@ -0,0 +1,31 @@ +import subprocess +import sys + + +def test_profile_does_not_load_heavy_analysis_stack(): + code = r''' +import sys +import pandas as pd +import framevitals as fv + +df = pd.DataFrame({"x": [1, 2, 3], "label": ["a", "b", "a"]}) +result = fv.profile(df) +assert result["shape"]["rows"] == 3 + +for name in ( + "framevitals.pipeline", + "framevitals.anomaly_ensemble", + "framevitals.deep_statistics_v2", + "sklearn", + "statsmodels", +): + assert name not in sys.modules, (name, sorted(k for k in sys.modules if k.startswith(name))) +''' + completed = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr diff --git a/tests/test_gate_api.py b/tests/test_gate_api.py new file mode 100644 index 0000000..70b4053 --- /dev/null +++ b/tests/test_gate_api.py @@ -0,0 +1,123 @@ +import pandas as pd +import pytest + +import framevitals as fv + + +def _reference() -> pd.DataFrame: + return pd.DataFrame({ + "age": list(range(20, 50)), + "plan": ["basic", "pro", "team"] * 10, + }) + + +def test_gate_requires_at_least_one_check(): + with pytest.raises(ValueError, match="reference=.*contract"): + fv.gate(_reference()) + + +def test_validation_only_gate_passes_clean_data(): + reference = _reference() + contract = fv.infer_contract(reference) + + result = fv.gate(reference.copy(), contract=contract) + + assert isinstance(result, fv.GateResult) + assert isinstance(result, dict) + assert result.status == "pass" + assert result.passed is True + assert result["checks_run"] == ["validation"] + assert result["checks"]["validation"]["status"] == "pass" + assert "FrameVitals quality gate" in result.summary_text() + + +def test_validation_warning_can_warn_or_be_promoted_to_failure(): + reference = _reference() + contract = fv.infer_contract(reference) + current = reference.copy() + current.loc[0, "plan"] = "enterprise" + + warning = fv.gate(current, contract=contract) + strict = fv.gate( + current, + contract=contract, + fail_on_validation_warning=True, + ) + + assert warning.status == "warn" + assert warning.passed is True + assert strict.status == "fail" + assert strict.passed is False + assert any("promoted to failure" in reason for reason in strict.reasons) + + +def test_drift_only_gate_uses_configurable_thresholds(): + reference = pd.DataFrame({"value": list(range(50))}) + current = pd.DataFrame({"value": list(range(100, 150))}) + + default = fv.gate(current, reference=reference) + strict = fv.gate( + current, + reference=reference, + drift_warn_on="minor", + drift_fail_on="moderate", + ) + + assert default.status in {"warn", "fail"} + assert strict.status == "fail" + assert strict.passed is False + assert strict["checks_run"] == ["drift"] + assert strict["checks"]["drift"]["available"] is True + + +def test_combined_gate_failure_wins_over_warning(): + reference = _reference() + contract = fv.infer_contract(reference, numeric_tolerance=0) + current = reference.copy() + current["age"] = current["age"] + 100 + current.loc[0, "plan"] = "enterprise" + + result = fv.gate( + current, + reference=reference, + contract=contract, + drift_warn_on="minor", + drift_fail_on="moderate", + ) + + assert result.status == "fail" + assert result.passed is False + assert set(result["checks_run"]) == {"validation", "drift"} + assert result["checks"]["validation"]["status"] in {"warn", "fail"} + assert result["checks"]["drift"]["available"] is True + assert result.reasons + + +def test_requested_but_unavailable_drift_fails_gate(): + reference = pd.DataFrame({"left": list(range(30))}) + current = pd.DataFrame({"right": list(range(30))}) + + result = fv.gate(current, reference=reference) + + assert result.status == "fail" + assert result.passed is False + assert result["checks"]["drift"]["available"] is False + assert any("could not produce" in reason for reason in result.reasons) + + +def test_gate_validates_threshold_order_and_max_columns(): + frame = _reference() + + with pytest.raises(ValueError, match="drift_warn_on"): + fv.gate(frame, reference=frame, drift_warn_on="extreme") + with pytest.raises(ValueError, match="drift_fail_on"): + fv.gate(frame, reference=frame, drift_fail_on="extreme") + with pytest.raises(ValueError, match="cannot be more severe"): + fv.gate( + frame, + reference=frame, + drift_warn_on="severe", + drift_fail_on="moderate", + ) + with pytest.raises(ValueError, match="max_columns"): + fv.gate(frame, reference=frame, max_columns=0) diff --git a/tests/test_higher_moments.py b/tests/test_higher_moments.py new file mode 100644 index 0000000..a6bf22a --- /dev/null +++ b/tests/test_higher_moments.py @@ -0,0 +1,58 @@ +import numpy as np +import pandas as pd +import pytest + +from framevitals.analysis_state import NumericColumnState +from framevitals.backends import numeric_state + + +def _reference_series() -> pd.Series: + return pd.Series([1.0, 2.0, 2.0, 3.0, 9.0, 12.0, np.nan, np.inf]) + + +def test_numpy_numeric_state_matches_pandas_bias_corrected_shape(): + series = _reference_series() + finite = series[np.isfinite(series.to_numpy(dtype="float64", na_value=np.nan))] + + state = numeric_state(series, backend="numpy") + + assert state["count"] == len(finite) + assert state["missing"] == 1 + assert state["infinite"] == 1 + assert state["skewness"] == pytest.approx(float(finite.skew()), abs=1e-12) + assert state["kurtosis"] == pytest.approx(float(finite.kurtosis()), abs=1e-12) + assert state["m3"] != 0.0 + assert state["m4"] > 0.0 + + +def test_numeric_column_state_partition_merge_preserves_shape_statistics(): + values = pd.Series( + np.concatenate([ + np.linspace(-8.0, 3.0, 2_500), + np.linspace(4.0, 40.0, 1_500) ** 1.15, + ]) + ) + full = NumericColumnState.from_series(values) + partitions = [ + NumericColumnState.from_series(chunk) + for chunk in np.array_split(values, 7) + ] + merged = NumericColumnState() + for partition in partitions: + merged = merged.merge(partition) + + assert merged.count == full.count + assert merged.mean == pytest.approx(full.mean, rel=1e-12, abs=1e-12) + assert merged.m2 == pytest.approx(full.m2, rel=1e-10, abs=1e-10) + assert merged.m3 == pytest.approx(full.m3, rel=1e-10, abs=1e-10) + assert merged.m4 == pytest.approx(full.m4, rel=1e-10, abs=1e-10) + assert merged.skewness == pytest.approx(float(values.skew()), rel=1e-10, abs=1e-10) + assert merged.kurtosis == pytest.approx(float(values.kurtosis()), rel=1e-10, abs=1e-10) + + +def test_constant_state_has_undefined_shape_statistics(): + state = NumericColumnState.from_series(pd.Series([5.0, 5.0, 5.0, 5.0, 5.0])) + + assert state.std == 0.0 + assert state.skewness is None + assert state.kurtosis is None diff --git a/tests/test_lazy_public_api.py b/tests/test_lazy_public_api.py new file mode 100644 index 0000000..09f2079 --- /dev/null +++ b/tests/test_lazy_public_api.py @@ -0,0 +1,41 @@ +import subprocess +import sys + + +def test_top_level_import_keeps_heavy_implementation_lazy(): + script = r''' +import sys +import framevitals + +required = ( + "profile", + "roles", + "health", + "ml_readiness", + "quality", + "statistics", + "anomalies", + "target_analysis", + "analyze", + "compare", + "validate", + "gate", +) +for name in required: + assert hasattr(framevitals, name), name + +assert "framevitals.api" not in sys.modules +assert "framevitals.pipeline" not in sys.modules +assert "framevitals.cleaning_plan" not in sys.modules +assert "pandas" not in sys.modules +assert "numpy" not in sys.modules +''' + + completed = subprocess.run( + [sys.executable, "-c", script], + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr diff --git a/tests/test_mixed_ground_truth_pipeline.py b/tests/test_mixed_ground_truth_pipeline.py new file mode 100644 index 0000000..9024944 --- /dev/null +++ b/tests/test_mixed_ground_truth_pipeline.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + +import framevitals as fv +from framevitals.fast_anomaly import fast_anomaly_scan +from framevitals.relationship_graph import build_numeric_relationship_graph +from framevitals.semantic_types import infer_semantic_types +from framevitals.target_intelligence import run_target_intelligence + + +def _mixed_frame(rows: int = 12_000) -> pd.DataFrame: + rng = np.random.default_rng(20260817) + linear = np.arange(rows, dtype=np.float64) + correlated = linear * 3.0 + 7.0 + noisy_signal = linear * 0.25 + rng.normal(scale=5.0, size=rows) + normal = rng.normal(loc=4.0, scale=1.5, size=rows) + exponential = rng.exponential(scale=2.0, size=rows) + + missing_numeric = normal.copy() + missing_numeric[::101] = np.nan + + category = np.array(["alpha", "beta", "gamma", "delta"] * (rows // 4), dtype=object) + category[::97] = None + + target = (linear % 10 >= 2).astype(np.int8) + + return pd.DataFrame({ + "linear": linear, + "correlated": correlated, + "noisy_signal": noisy_signal, + "normal": normal, + "exponential": exponential, + "missing_numeric": missing_numeric, + "category": category, + "event_time": pd.date_range("2026-01-01", periods=rows, freq="min"), + "target": target, + }) + + +def _write_mixed_parquet(path, rows: int = 12_000) -> pd.DataFrame: + frame = _mixed_frame(rows) + pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=777) + return frame + + +def test_mixed_parquet_profile_matches_ground_truth(tmp_path): + path = tmp_path / "mixed-ground-truth.parquet" + frame = _write_mixed_parquet(path) + + result = fv.profile(path) + + assert result["shape"] == {"rows": len(frame), "columns": len(frame.columns)} + assert result["streaming_metadata"]["enabled"] is True + assert result["streaming_metadata"]["full_materialization"] is False + assert result["missing_counts"]["missing_numeric"] == int(frame["missing_numeric"].isna().sum()) + assert result["missing_counts"]["category"] == int(frame["category"].isna().sum()) + assert result["numeric_summary"]["linear"]["mean"] == pytest.approx( + round(float(frame["linear"].mean()), 3) + ) + assert result["numeric_summary"]["correlated"]["min"] == pytest.approx(7.0) + assert result["numeric_summary"]["correlated"]["max"] == pytest.approx( + float(frame["correlated"].max()) + ) + assert result["numeric_summary"]["linear"]["50%"] == pytest.approx( + float(frame["linear"].median()), rel=0.03 + ) + assert "category" in result["categorical_summary"] + assert "event_time" in result["date_columns"] + + +def test_mixed_parquet_full_public_analysis_keeps_streaming_and_target_intelligence(tmp_path): + path = tmp_path / "mixed-analysis.parquet" + frame = _write_mixed_parquet(path) + + result = fv.analyze( + path, + target="target", + mode="quick", + artifacts=False, + workers=2, + ) + + assert result["profile"]["shape"] == {"rows": len(frame), "columns": len(frame.columns)} + streaming = result["execution"]["streaming"] + assert streaming["enabled"] is True + assert streaming["full_materialization"] is False + assert streaming["single_full_source_profile_scan"] is True + assert result["target_intelligence"]["available"] is True + assert result["target_intelligence"]["target_column"] == "target" + assert result["target_intelligence"]["task_type"] == "classification" + assert 0 <= result["health"]["overall_score"] <= 100 + assert 0 <= result["ml_readiness"]["score"] <= 100 + + +def test_known_numeric_relationship_is_found_without_dense_matrix(): + frame = _mixed_frame(2_000)[["linear", "correlated", "noisy_signal", "normal"]] + result = build_numeric_relationship_graph( + frame, + max_sample_rows=512, + min_abs_correlation=0.98, + ) + + pairs = { + frozenset((edge["source"], edge["target"])) + for edge in result["edges"] + } + assert frozenset(("linear", "correlated")) in pairs + assert result["candidate_generation"]["candidate_pairs"] < result["candidate_generation"][ + "total_possible_dense_pairs" + ] + + +def test_injected_multivariate_anomalies_have_high_recall(): + rng = np.random.default_rng(77) + rows = 3_000 + matrix = rng.normal(size=(rows, 10)) + injected = set(range(rows - 16, rows)) + matrix[-16:] += 10.0 + frame = pd.DataFrame(matrix, columns=[f"x{index}" for index in range(matrix.shape[1])]) + + result = fast_anomaly_scan(frame, contamination=0.02, top_k=30) + top_indices = {int(item["row_index"]) for item in result["top_rows"][:25]} + + assert result["available"] is True + assert len(top_indices & injected) >= 12 + + +def test_target_intelligence_finds_numeric_and_categorical_leakage(): + target = [0, 1] * 200 + frame = pd.DataFrame({ + "numeric_signal": [value * 10 + (index % 3) for index, value in enumerate(target)], + "segment": ["stay" if value == 0 else "leave" for value in target], + "noise": np.random.default_rng(9).normal(size=len(target)), + "target": target, + }) + + result = run_target_intelligence(frame, target_column="target") + associations = {item["feature"]: item for item in result["top_associations"]} + leakage = {item["feature"] for item in result["leakage"]["warnings"]} + + assert result["task_type"] == "classification" + assert associations["numeric_signal"]["score"] > 0.9 + assert associations["segment"]["score"] > 0.9 + assert "segment" in leakage + + +def test_semantic_type_ground_truth_for_sensitive_string_patterns(): + cases = { + "email": pd.Series([f"user{index}@example.com" for index in range(40)]), + "url": pd.Series([f"https://example.com/{index}" for index in range(40)]), + "ip_address": pd.Series([f"10.0.0.{index + 1}" for index in range(40)]), + } + + for expected, series in cases.items(): + result = infer_semantic_types(series, max_samples=40) + assert result["primary"] == expected + assert result["candidates"][0]["confidence"] >= 0.7 diff --git a/tests/test_ml_preprocessing_v2.py b/tests/test_ml_preprocessing_v2.py new file mode 100644 index 0000000..edbc4f5 --- /dev/null +++ b/tests/test_ml_preprocessing_v2.py @@ -0,0 +1,103 @@ +import numpy as np +import pandas as pd +import pytest + +from framevitals.ml_preprocessing import prepare_ml_matrix + + +def test_id_detection_uses_boundaries_not_raw_substrings(): + rows = 30 + df = pd.DataFrame({ + "paid_amount": [100 + index for index in range(rows)], + "width": [10 + (index % 5) for index in range(rows)], + "customer_id": [f"CUST-{index:03d}" for index in range(rows)], + "candidate_score": [50 + (index % 10) for index in range(rows)], + "target": [0, 1] * 15, + }) + + result = prepare_ml_matrix(df, target="target") + dropped = {item["column"]: item["reason"] for item in result["dropped_columns"]} + + assert "paid_amount" in result["numeric_features"] + assert "width" in result["numeric_features"] + assert "candidate_score" in result["numeric_features"] + assert dropped["customer_id"] == "id_like_name" + + +def test_numeric_infinities_are_replaced_for_imputation(): + rows = 30 + df = pd.DataFrame({ + "measurement": [float(index) for index in range(rows)], + "target": [0, 1] * 15, + }) + df.loc[3, "measurement"] = np.inf + df.loc[7, "measurement"] = -np.inf + + result = prepare_ml_matrix(df, target="target") + + assert result["usable"] is True + assert result["infinite_values_replaced"] == {"measurement": 2} + assert int(result["X"]["measurement"].isna().sum()) == 2 + assert any("infinite values" in warning for warning in result["warnings"]) + + +def test_infinite_numeric_targets_are_dropped_as_invalid_labels(): + rows = 30 + target = [float(index) for index in range(rows)] + target[-1] = np.inf + df = pd.DataFrame({ + "feature": list(range(rows)), + "target": target, + }) + + result = prepare_ml_matrix(df, target="target") + + assert result["target_infinite_values_dropped"] == 1 + assert len(result["y"]) == 29 + assert np.isfinite(result["y"].to_numpy(dtype=float)).all() + + +def test_high_cardinality_categorical_is_bounded_before_one_hot_encoding(): + rows = 300 + df = pd.DataFrame({ + "category": [f"group-{index % 250}" for index in range(rows)], + "feature": [index % 12 for index in range(rows)], + "target": [0, 1] * 150, + }) + + result = prepare_ml_matrix( + df, + target="target", + max_categorical_levels=100, + ) + dropped = {item["column"]: item["reason"] for item in result["dropped_columns"]} + + assert dropped["category"] == "high_cardinality_categorical" + assert "category" not in result["categorical_features"] + assert "feature" in result["numeric_features"] + + +def test_real_time_columns_are_excluded_but_unrelated_names_are_kept(): + rows = 30 + df = pd.DataFrame({ + "event_date": pd.date_range("2026-01-01", periods=rows).astype(str), + "candidate_label": ["A", "B", "C"] * 10, + "target": [0, 1] * 15, + }) + + result = prepare_ml_matrix(df, target="target") + dropped = {item["column"]: item["reason"] for item in result["dropped_columns"]} + + assert dropped["event_date"] == "time_like_non_numeric" + assert "candidate_label" in result["categorical_features"] + + +def test_preprocessing_control_validation(): + df = pd.DataFrame({"x": list(range(20)), "target": [0, 1] * 10}) + + with pytest.raises(ValueError, match="drop_high_unique_ratio"): + prepare_ml_matrix(df, "target", drop_high_unique_ratio=0) + with pytest.raises(ValueError, match="min_non_missing"): + prepare_ml_matrix(df, "target", min_non_missing=0) + with pytest.raises(ValueError, match="max_categorical_levels"): + prepare_ml_matrix(df, "target", max_categorical_levels=1) diff --git a/tests/test_model_leaderboard_v2.py b/tests/test_model_leaderboard_v2.py new file mode 100644 index 0000000..5fe1ca3 --- /dev/null +++ b/tests/test_model_leaderboard_v2.py @@ -0,0 +1,143 @@ +import pandas as pd +import pytest +from sklearn.dummy import DummyClassifier, DummyRegressor +from sklearn.linear_model import LogisticRegression, Ridge + +import framevitals.model_leaderboard as leaderboard_module + + +def _classification_frame(labels=(10, 20), rows=40): + return pd.DataFrame({ + "feature": list(range(rows)), + "segment": ["A", "B"] * (rows // 2), + "target": [labels[index % len(labels)] for index in range(rows)], + }) + + +def test_non_consecutive_numeric_class_labels_are_encoded_safely(monkeypatch): + def registry(class_count): + return { + "DummyClassifier": DummyClassifier(strategy="most_frequent"), + "LogisticRegression": LogisticRegression(max_iter=500, random_state=42), + } + + monkeypatch.setattr(leaderboard_module, "_classification_registry", registry) + result = leaderboard_module.run_model_leaderboard( + _classification_frame(labels=(10, 20)), + target_column="target", + task_type="classification", + n_splits=2, + ) + + assert result["available"] is True + assert result["target_encoding"] == [ + {"encoded": 0, "label": 10}, + {"encoded": 1, "label": 20}, + ] + assert result["winner"] is not None + assert result["models_succeeded"] >= 2 + assert result["baseline"]["model"] == "DummyClassifier" + + +def test_rare_class_returns_clear_cv_unavailable_result(monkeypatch): + df = pd.DataFrame({ + "feature": list(range(25)), + "target": [0] * 24 + [1], + }) + + def registry(class_count): + return {"LogisticRegression": LogisticRegression(max_iter=200)} + + monkeypatch.setattr(leaderboard_module, "_classification_registry", registry) + result = leaderboard_module.run_model_leaderboard( + df, + target_column="target", + task_type="classification", + n_splits=5, + ) + + assert result["available"] is False + assert "at least 2 rows" in result["message"] + + +def test_regression_cv_caps_requested_folds_and_reports_baseline(monkeypatch): + rows = 24 + df = pd.DataFrame({ + "feature": list(range(rows)), + "target": [float(index * 3 + 1) for index in range(rows)], + }) + + def registry(): + return { + "DummyRegressor": DummyRegressor(strategy="mean"), + "Ridge": Ridge(alpha=1.0), + } + + monkeypatch.setattr(leaderboard_module, "_regression_registry", registry) + result = leaderboard_module.run_model_leaderboard( + df, + target_column="target", + task_type="regression", + n_splits=100, + ) + + assert result["available"] is True + assert result["cv"]["requested_splits"] == 100 + # R2 needs at least two observations in each test fold, so the safe upper + # bound is floor(n_rows / 2), not one fold per row. + assert result["cv"]["actual_splits"] == rows // 2 + assert result["baseline"]["model"] == "DummyRegressor" + assert result["winner"]["model"] == "Ridge" + assert result["winner"]["beats_baseline"] is True + assert result["winner"]["lift_over_baseline"] is not None + + +def test_leaderboard_records_model_failures_without_sinking_other_models(monkeypatch): + class BrokenEstimator: + def get_params(self, deep=True): + return {} + + def fit(self, X, y): + raise RuntimeError("broken intentionally") + + def registry(class_count): + return { + "Broken": BrokenEstimator(), + "LogisticRegression": LogisticRegression(max_iter=500), + } + + monkeypatch.setattr(leaderboard_module, "_classification_registry", registry) + result = leaderboard_module.run_model_leaderboard( + _classification_frame(labels=(0, 1)), + target_column="target", + task_type="classification", + n_splits=2, + ) + + assert result["available"] is True + assert result["winner"]["model"] == "LogisticRegression" + assert result["models_failed"] == 1 + assert result["model_failures"][0]["model"] == "Broken" + + +def test_leaderboard_validates_controls(): + df = _classification_frame(labels=(0, 1)) + + with pytest.raises(ValueError, match="task_type"): + leaderboard_module.run_model_leaderboard( + df, + target_column="target", + task_type="ranking", + ) + with pytest.raises(ValueError, match="n_splits"): + leaderboard_module.run_model_leaderboard( + df, + target_column="target", + n_splits=1, + ) + with pytest.raises(ValueError, match="n_jobs"): + leaderboard_module.run_model_leaderboard( + df, + target_column="target", + n_jobs=0, + ) diff --git a/tests/test_native_arrow_batch_profile.py b/tests/test_native_arrow_batch_profile.py new file mode 100644 index 0000000..e3e4fb2 --- /dev/null +++ b/tests/test_native_arrow_batch_profile.py @@ -0,0 +1,39 @@ +import pytest + +pa = pytest.importorskip("pyarrow") +native = pytest.importorskip("framevitals._native") + + +def test_native_arrow_batch_accumulator_profiles_int16_without_numpy_bridge(): + accumulator_type = getattr(native, "ArrowBatchProfileAccumulator", None) + assert accumulator_type is not None + + batch = pa.record_batch( + [ + pa.array([1, 2, None, 4], type=pa.int16()), + pa.array([10.0, 20.0, 30.0, 40.0], type=pa.float64()), + pa.array(["a", "b", "c", "d"]), + ], + names=["small", "wide", "label"], + ) + accumulator = accumulator_type() + accumulator.update(batch) + payload = dict(accumulator.snapshot()) + profiles = { + str(name): dict(value) + for name, value in dict(payload["profiles"]).items() + } + + assert payload["rows"] == 4 + assert payload["sketch_policy"] == "moments_and_log_quantiles" + assert profiles["small"]["count"] == 3 + assert profiles["small"]["missing"] == 1 + assert profiles["small"]["mean"] == pytest.approx(7 / 3) + assert profiles["small"]["minimum"] == 1.0 + assert profiles["small"]["maximum"] == 4.0 + assert profiles["small"]["sketch_policy"] == "moments_and_log_quantiles" + assert "heavy_hitters" not in profiles["small"] + assert "reservoir" not in profiles["small"] + assert profiles["small"]["quantiles"]["p50"] is not None + assert profiles["wide"]["mean"] == pytest.approx(25.0) + assert "label" in payload["skipped_columns"] diff --git a/tests/test_native_profiler_routing.py b/tests/test_native_profiler_routing.py new file mode 100644 index 0000000..99ef338 --- /dev/null +++ b/tests/test_native_profiler_routing.py @@ -0,0 +1,120 @@ +import pandas as pd +import pytest + +import framevitals.profiler as profiler + + +def _frame() -> pd.DataFrame: + return pd.DataFrame({ + "value": [1.0, 2.0, None, 4.0], + "other": [10.0, 20.0, 30.0, 40.0], + "label": ["a", "b", "a", "b"], + }) + + +def test_small_profile_keeps_exact_pandas_summary_when_native_is_available(monkeypatch): + monkeypatch.delenv("FRAMEVITALS_BACKEND", raising=False) + monkeypatch.setattr(profiler, "resolve_numeric_backend", lambda: "rust") + + result = profiler.build_profile(_frame()) + + metadata = result["numeric_summary_metadata"] + assert metadata["backend"] == "pandas" + assert metadata["approximate_quantiles"] is False + assert metadata["native_eligible"] is True + assert metadata["native_threshold_reached"] is False + assert result["numeric_summary"]["value"]["50%"] == pytest.approx(2.0) + + +def test_forced_native_profile_reuses_missing_counts_and_preserves_summary_shape(monkeypatch): + monkeypatch.setenv("FRAMEVITALS_BACKEND", "rust") + monkeypatch.setattr(profiler, "resolve_numeric_backend", lambda: "rust") + + calls = [] + + def fake_numeric_profile(series, *, backend, stream_id): + calls.append((series.name, backend, stream_id)) + if series.name == "value": + return { + "count": 3, + "missing": 1, + "infinite": 0, + "mean": 7.0 / 3.0, + "std": 1.527525, + "minimum": 1.0, + "maximum": 4.0, + "quantiles": { + "p25": 1.5, + "p50": 2.0, + "p75": 3.0, + "relative_accuracy": 0.01, + }, + } + return { + "count": 4, + "missing": 0, + "infinite": 0, + "mean": 25.0, + "std": 12.909944, + "minimum": 10.0, + "maximum": 40.0, + "quantiles": { + "p25": 17.5, + "p50": 25.0, + "p75": 32.5, + "relative_accuracy": 0.01, + }, + } + + monkeypatch.setattr(profiler, "numeric_profile", fake_numeric_profile) + result = profiler.build_profile(_frame()) + + assert calls == [("value", "rust", 0), ("other", "rust", 1)] + assert result["missing_counts"]["value"] == 1 + assert result["missing_counts"]["other"] == 0 + assert set(result["numeric_summary"]["value"]) == { + "count", + "mean", + "std", + "min", + "25%", + "50%", + "75%", + "max", + } + metadata = result["numeric_summary_metadata"] + assert metadata["backend"] == "rust" + assert metadata["approximate_quantiles"] is True + assert metadata["quantile_relative_accuracy"] == 0.01 + assert metadata["raw_observations_retained"] is False + + +def test_auto_native_failure_falls_back_and_discloses_reason(monkeypatch): + monkeypatch.delenv("FRAMEVITALS_BACKEND", raising=False) + monkeypatch.setattr(profiler, "resolve_numeric_backend", lambda: "rust") + monkeypatch.setattr(profiler, "NATIVE_NUMERIC_PROFILE_MIN_ROWS", 1) + monkeypatch.setattr( + profiler, + "numeric_profile", + lambda *args, **kwargs: (_ for _ in ()).throw(BufferError("not contiguous")), + ) + + result = profiler.build_profile(_frame()) + metadata = result["numeric_summary_metadata"] + + assert metadata["backend"] == "pandas" + assert metadata["fallback_from"] == "rust" + assert "BufferError" in metadata["fallback_reason"] + + +def test_forced_native_failure_is_not_silently_hidden(monkeypatch): + monkeypatch.setenv("FRAMEVITALS_BACKEND", "rust") + monkeypatch.setattr(profiler, "resolve_numeric_backend", lambda: "rust") + monkeypatch.setattr( + profiler, + "numeric_profile", + lambda *args, **kwargs: (_ for _ in ()).throw(BufferError("bad buffer")), + ) + + with pytest.raises(BufferError, match="bad buffer"): + profiler.build_profile(_frame()) diff --git a/tests/test_neural_anomaly.py b/tests/test_neural_anomaly.py new file mode 100644 index 0000000..53235ba --- /dev/null +++ b/tests/test_neural_anomaly.py @@ -0,0 +1,66 @@ +import numpy as np +import pandas as pd + +from framevitals.budgeted_analysis import run_budgeted_anomalies +from framevitals.execution import derive_execution_budget +from framevitals.neural_anomaly import neural_reconstruction_anomalies + + +def _frame(rows: int = 800) -> pd.DataFrame: + rng = np.random.default_rng(42) + x = rng.normal(size=(rows, 5)) + x[-5:] += 8.0 + frame = pd.DataFrame(x, columns=[f"x{i}" for i in range(5)]) + frame["constant"] = 1.0 + return frame + + +def test_neural_reconstruction_detector_is_bounded_and_finds_shifted_rows(): + frame = _frame() + result = neural_reconstruction_anomalies( + frame, + max_rows=500, + max_columns=5, + max_iter=20, + top_k=15, + ) + + assert result["available"] is True + assert result["sample_rows"] <= 500 + assert result["columns_used"] <= 5 + assert result["method"] == "bounded_mlp_reconstruction" + assert result["architecture"][0] == result["architecture"][-1] + assert result["top_rows"] + + +def test_neural_reconstruction_is_research_only(monkeypatch): + seen = {"calls": 0} + + def fake_classical(frame, **kwargs): + return {"available": True, "n_rows_scored": len(frame)} + + def fake_neural(frame, **kwargs): + seen["calls"] += 1 + return {"available": True, "method": "fake_neural"} + + monkeypatch.setattr( + "framevitals.budgeted_analysis.detect_anomalies_ensemble", + fake_classical, + ) + monkeypatch.setattr( + "framevitals.budgeted_analysis.neural_reconstruction_anomalies", + fake_neural, + ) + + frame = _frame(200) + deep = derive_execution_budget(len(frame), len(frame.columns), mode="deep") + deep_result = run_budgeted_anomalies(frame, budget=deep) + assert seen["calls"] == 0 + assert "neural_reconstruction" not in deep_result + assert deep_result["execution"]["neural_reconstruction_enabled"] is False + + research = derive_execution_budget(len(frame), len(frame.columns), mode="research") + research_result = run_budgeted_anomalies(frame, budget=research) + assert seen["calls"] == 1 + assert research_result["neural_reconstruction"]["available"] is True + assert research_result["execution"]["neural_reconstruction_enabled"] is True diff --git a/tests/test_operation_import_boundaries.py b/tests/test_operation_import_boundaries.py new file mode 100644 index 0000000..111e982 --- /dev/null +++ b/tests/test_operation_import_boundaries.py @@ -0,0 +1,38 @@ +import subprocess +import sys + + +def test_contract_and_drift_operations_do_not_load_full_pipeline(): + code = r''' +import sys +import pandas as pd +import framevitals as fv + +reference = pd.DataFrame({"x": range(30), "label": ["a", "b"] * 15}) +current = reference.copy() + +contract = fv.infer_contract(reference) +validation = fv.validate(current, contract) +drift = fv.compare(reference, current) +gate = fv.gate(current, reference=reference, contract=contract) + +assert validation["status"] in {"pass", "warn"} +assert drift["available"] is True +assert gate["passed"] is True + +for name in ( + "framevitals.pipeline", + "framevitals.anomaly_ensemble", + "framevitals.deep_statistics_v2", + "framevitals.model_leaderboard", +): + assert name not in sys.modules, name +''' + completed = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr diff --git a/tests/test_optional_dependency_boundaries.py b/tests/test_optional_dependency_boundaries.py new file mode 100644 index 0000000..93c34df --- /dev/null +++ b/tests/test_optional_dependency_boundaries.py @@ -0,0 +1,66 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from framevitals.loader import load_dataset + + +def _run_import_assertion(code: str): + src_root = Path(__file__).resolve().parents[1] / "src" + env = os.environ.copy() + env["PYTHONPATH"] = str(src_root) + return subprocess.run( + [sys.executable, "-c", code], + cwd=src_root.parent, + env=env, + capture_output=True, + text=True, + check=False, + ) + + +def test_core_import_does_not_eagerly_load_ai_dependency(): + completed = _run_import_assertion( + "import sys; import framevitals; assert 'pydantic' not in sys.modules" + ) + assert completed.returncode == 0, completed.stderr + + +def test_pipeline_import_does_not_eagerly_load_plotting_stack(): + completed = _run_import_assertion( + ( + "import sys; import framevitals.pipeline; " + "assert 'framevitals.visualizer' not in sys.modules; " + "assert 'framevitals.explainability' not in sys.modules; " + "assert 'matplotlib' not in sys.modules; " + "assert 'seaborn' not in sys.modules" + ) + ) + assert completed.returncode == 0, completed.stderr + + +def test_explainability_import_does_not_require_plotting_stack(): + completed = _run_import_assertion( + ( + "import sys; import framevitals.explainability; " + "assert 'matplotlib' not in sys.modules; " + "assert 'seaborn' not in sys.modules" + ) + ) + assert completed.returncode == 0, completed.stderr + + +def test_excel_loader_reports_framevitals_extra(monkeypatch, tmp_path): + path = tmp_path / "dataset.xlsx" + path.write_bytes(b"placeholder") + + def missing_engine(*args, **kwargs): + raise ImportError("missing Excel engine") + + monkeypatch.setattr("framevitals.loader.pd.read_excel", missing_engine) + + with pytest.raises(ImportError, match=r"framevitals\[excel\]"): + load_dataset(path) diff --git a/tests/test_parquet_source_cache.py b/tests/test_parquet_source_cache.py new file mode 100644 index 0000000..64c7fdb --- /dev/null +++ b/tests/test_parquet_source_cache.py @@ -0,0 +1,43 @@ +import pandas as pd +import pytest + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + +from framevitals.sources import ParquetSource + + +def test_parquet_source_reuses_one_parquet_file_handle(tmp_path, monkeypatch): + path = tmp_path / "cache.parquet" + frame = pd.DataFrame({ + "x": range(100), + "y": [value * 2 for value in range(100)], + }) + pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=17) + + real_parquet_file = pq.ParquetFile + calls = {"count": 0} + + def counted_parquet_file(*args, **kwargs): + calls["count"] += 1 + return real_parquet_file(*args, **kwargs) + + monkeypatch.setattr(pq, "ParquetFile", counted_parquet_file) + + source = ParquetSource(path) + first_metadata = source.inspect() + second_metadata = source.inspect() + first_schema = source.schema() + second_schema = source.schema() + first_batches = list(source.iter_batches(batch_size=23, columns=["x"])) + second_batches = list(source.iter_batches(batch_size=31, columns=["y"])) + + assert calls["count"] == 1 + assert first_metadata is second_metadata + assert first_schema is second_schema + assert first_metadata.rows == 100 + assert first_metadata.columns == 2 + assert sum(batch.num_rows for batch in first_batches) == 100 + assert sum(batch.num_rows for batch in second_batches) == 100 + assert first_batches[0].schema.names == ["x"] + assert second_batches[0].schema.names == ["y"] diff --git a/tests/test_parquet_streaming.py b/tests/test_parquet_streaming.py new file mode 100644 index 0000000..17ce1da --- /dev/null +++ b/tests/test_parquet_streaming.py @@ -0,0 +1,349 @@ +import numpy as np +import pandas as pd +import pytest + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + +import framevitals +from framevitals.sources import ParquetSource, resolve_source + + +def _write_parquet(path, rows: int = 10_000) -> pd.DataFrame: + frame = pd.DataFrame({ + "value": np.arange(rows, dtype=np.float64), + "other": np.arange(rows, dtype=np.float64) * 2.0 + 3.0, + "group": [f"g-{index % 7}" for index in range(rows)], + "event_time": pd.date_range("2026-01-01", periods=rows, freq="min"), + }) + frame.loc[::101, "value"] = np.nan + table = pa.Table.from_pandas(frame, preserve_index=False) + pq.write_table(table, path, row_group_size=777) + return frame + + +def test_parquet_source_exposes_metadata_projection_and_batches(tmp_path): + path = tmp_path / "dataset.parquet" + frame = _write_parquet(path, rows=2_500) + + source = resolve_source(path) + assert isinstance(source, ParquetSource) + + metadata = source.inspect() + assert metadata.rows == len(frame) + assert metadata.columns == len(frame.columns) + assert metadata.format == "parquet" + assert metadata.materialized is False + assert metadata.supports_projection is True + assert metadata.supports_streaming is True + + batches = list(source.iter_batches(batch_size=600, columns=["value", "group"])) + assert sum(batch.num_rows for batch in batches) == len(frame) + assert batches[0].schema.names == ["value", "group"] + assert max(batch.num_rows for batch in batches) <= 600 + + +def test_public_profile_streams_parquet_without_calling_load(tmp_path, monkeypatch): + path = tmp_path / "stream.parquet" + frame = _write_parquet(path, rows=12_000) + + def fail_load(self): + raise AssertionError("streaming profile must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.profile(path) + + assert result["dataset_name"] == "stream.parquet" + assert result["shape"] == {"rows": len(frame), "columns": 4} + assert result["streaming_metadata"]["enabled"] is True + assert result["streaming_metadata"]["full_materialization"] is False + assert result["streaming_metadata"]["sample_rows"] == len(frame) + assert result["source_metadata"]["supports_streaming"] is True + assert result["missing_counts"]["value"] == int(frame["value"].isna().sum()) + assert result["numeric_summary"]["value"]["count"] == int(frame["value"].notna().sum()) + assert result["numeric_summary"]["other"]["mean"] == pytest.approx( + round(float(frame["other"].mean()), 3) + ) + assert "group" in result["categorical_summary"] + assert "event_time" in result["date_columns"] + + +def test_public_analyze_streams_parquet_without_calling_load(tmp_path, monkeypatch): + path = tmp_path / "analyze.parquet" + frame = _write_parquet(path, rows=12_000) + + def fail_load(self): + raise AssertionError("analyze must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.analyze(path, mode="quick", artifacts=False, workers=1) + + assert result["filename"] == "analyze.parquet" + assert result["profile"]["shape"] == {"rows": len(frame), "columns": 4} + assert result["dataset_signals"]["row_count"] == len(frame) + assert result["execution"]["streaming"]["enabled"] is True + assert result["execution"]["streaming"]["full_materialization"] is False + assert result["execution"]["streaming"]["source_rows"] == len(frame) + assert result["execution"]["streaming"]["working_sample_rows"] == 5_000 + assert result["execution"]["streaming"]["single_full_source_profile_scan"] is True + assert result["execution"]["module_scope"]["profile"] == "full_stream" + assert result["cleaning"]["streaming_status"] == "deferred_streaming" + assert result["charts"] == [] + assert result["config"]["artifacts"] is False + + +def test_public_roles_stream_parquet_without_calling_load(tmp_path, monkeypatch): + path = tmp_path / "roles.parquet" + frame = _write_parquet(path, rows=12_000) + + def fail_load(self): + raise AssertionError("roles must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.roles(path) + + assert result["dataset_name"] == "roles.parquet" + execution = result["execution"] + assert execution["full_materialization"] is False + assert execution["source_rows"] == len(frame) + assert execution["sample_rows"] == 5_000 + assert execution["sampled"] is True + + value = result["columns"]["value"] + expected_missing = int(frame["value"].isna().sum()) / len(frame) * 100 + assert value["missing_percent"] == pytest.approx(round(expected_missing, 2)) + assert value["source_rows"] == len(frame) + assert value["sample_rows"] == 5_000 + assert value["cardinality_scope"] == "bounded_row_sample" + assert value["cardinality_approximate"] is True + + group = result["columns"]["group"] + assert group["unique_count"] == 7 + assert "low_cardinality" in group["roles"] + assert group["cardinality_scope"] in { + "bounded_row_sample", + "full_stream_approximate", + } + + +def test_large_parquet_retains_only_bounded_row_sample(tmp_path): + path = tmp_path / "large.parquet" + frame = _write_parquet(path, rows=60_000) + + result = framevitals.profile(path) + + streaming = result["streaming_metadata"] + assert streaming["full_materialization"] is False + assert streaming["sample_rows"] == 50_000 + assert streaming["sample_strategy"] == "stratified_jitter_global_rows" + + categorical = result["categorical_summary_metadata"] + if categorical["native_full_stream_columns"]: + assert categorical["sampled"] is False + assert categorical["method"] == "native_full_stream_sketch" + assert "group" in categorical["native_full_stream_columns"] + assert categorical["sample_fallback_columns"] == [] + else: + assert categorical["sampled"] is True + assert categorical["method"] == "stratified_jitter_row_sample" + assert "group" in categorical["sample_fallback_columns"] + + assert result["correlation_metadata"]["row_sampled"] is True + assert result["duplicate_metadata"]["sampled"] is True + assert result["missing_counts"]["value"] == int(frame["value"].isna().sum()) + + +def test_public_health_streams_parquet_without_calling_load(tmp_path, monkeypatch): + path = tmp_path / "health.parquet" + frame = _write_parquet(path, rows=12_000) + + def fail_load(self): + raise AssertionError("health must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.health(path) + + assert result["dataset_name"] == "health.parquet" + assert 0 <= result["overall_score"] <= 100 + assert result["execution"]["method"] == "streaming_profile_with_bounded_row_sample" + assert result["execution"]["full_materialization"] is False + assert result["execution"]["source_rows"] == len(frame) + assert result["execution"]["sample_rows"] == len(frame) + assert result["execution"]["components"]["completeness"] == "full_stream_exact" + assert result["execution"]["components"]["outlier_safety"] == "exact" + expected_missing = int(frame["value"].isna().sum()) / (len(frame) * 4) * 100 + assert result["details"]["missing_percent"] == pytest.approx( + round(expected_missing, 2) + ) + + +def test_public_ml_readiness_streams_parquet_without_calling_load(tmp_path, monkeypatch): + path = tmp_path / "ml-readiness.parquet" + frame = _write_parquet(path, rows=12_000) + + def fail_load(self): + raise AssertionError("ML readiness must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.ml_readiness(path) + + assert result["dataset_name"] == "ml-readiness.parquet" + assert result["numeric_columns"] == ["value", "other"] + assert result["categorical_columns"] == ["group"] + expected_missing = int(frame["value"].isna().sum()) / (len(frame) * 4) * 100 + assert result["issues"]["missing_percent"] == pytest.approx( + round(expected_missing, 2) + ) + execution = result["execution"] + assert execution["method"] == "streaming_profile" + assert execution["full_materialization"] is False + assert execution["source_rows"] == len(frame) + assert execution["sample_rows"] == len(frame) + assert execution["components"]["missingness"] == "full_stream_exact" + assert execution["components"]["column_groups"] == "schema_exact" + assert execution["components"]["duplicate_rate"] == "exact" + assert 0 <= result["score"] <= 100 + + +def test_large_public_ml_readiness_discloses_duplicate_estimate(tmp_path, monkeypatch): + path = tmp_path / "large-ml-readiness.parquet" + frame = _write_parquet(path, rows=60_000) + + def fail_load(self): + raise AssertionError("large ML readiness must stay on the streaming path") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.ml_readiness(path) + + execution = result["execution"] + assert execution["full_materialization"] is False + assert execution["source_rows"] == len(frame) + assert execution["sample_rows"] == 50_000 + assert execution["components"]["duplicate_rate"] == "bounded_row_sample_estimate" + + +def test_large_public_health_discloses_bounded_outlier_estimate(tmp_path, monkeypatch): + path = tmp_path / "large-health.parquet" + frame = _write_parquet(path, rows=60_000) + + def fail_load(self): + raise AssertionError("large health must stay on the streaming path") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.health(path) + + execution = result["execution"] + assert execution["full_materialization"] is False + assert execution["source_rows"] == len(frame) + assert execution["sample_rows"] == 50_000 + assert execution["components"]["outlier_safety"] == "bounded_row_sample_estimate" + assert execution["components"]["uniqueness"] == "full_stream_sample_estimate" + + +def test_public_quality_streams_parquet_and_marks_sample_candidates(tmp_path, monkeypatch): + path = tmp_path / "quality.parquet" + frame = _write_parquet(path, rows=12_000) + frame["other_copy"] = frame["other"] + pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=777) + + def fail_load(self): + raise AssertionError("quality must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.quality(path, max_sample_rows=1_000) + + assert result["dataset_name"] == "quality.parquet" + assert result["rows"] == len(frame) + assert result["columns"] == len(frame.columns) + execution = result["execution"] + assert execution["full_materialization"] is False + assert execution["sampled"] is True + assert execution["sample_rows"] == 1_000 + assert set(execution["candidate_only_checks"]) == { + "primary_key_candidates", + "duplicate_columns", + } + + duplicate_candidates = result["duplicate_columns"] + assert duplicate_candidates + candidate = next( + item + for item in duplicate_candidates + if {item["canonical_column"], *item["duplicate_columns"]} + >= {"other", "other_copy"} + ) + assert candidate["candidate_only"] is True + assert candidate["confirmed_with_full_equality"] is False + assert candidate["confirmation_scope"] == "bounded_row_sample" + + +def test_public_anomalies_stream_numeric_projection_without_calling_load(tmp_path, monkeypatch): + path = tmp_path / "anomalies.parquet" + frame = _write_parquet(path, rows=12_000) + + def fail_load(self): + raise AssertionError("anomalies must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.anomalies( + path, + mode="quick", + max_columns=2, + top_k=5, + ) + + execution = result["execution"] + assert execution["full_materialization"] is False + assert execution["source_rows"] == len(frame) + assert execution["source_columns"] == 4 + assert execution["projected_columns"] == 2 + assert execution["sample_rows"] == 5_000 + assert execution["sampled"] is True + assert execution["strategy"] == "streaming_stratified_jitter_numeric_projection" + assert result["source"]["supports_projection"] is True + + +def test_plan_reads_only_bounded_parquet_sample_and_uses_true_shape(tmp_path, monkeypatch): + path = tmp_path / "plan.parquet" + frame = _write_parquet(path, rows=12_000) + + def fail_load(self): + raise AssertionError("plan must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.plan(path, mode="standard") + + assert result["shape"] == {"rows": len(frame), "columns": 4} + assert result["signals"]["row_count"] == len(frame) + assert result["signals"]["column_count"] == 4 + assert result["execution_budget"]["rows"] == len(frame) + planning = result["planning_data"] + assert planning["materialized_full_dataset"] is False + assert planning["full_scan"] is False + assert planning["sampled"] is True + assert planning["sample_rows"] == 5_000 + assert result["source"]["supports_streaming"] is True + + +def test_relationships_stream_numeric_projection_without_full_materialization( + tmp_path, + monkeypatch, +): + path = tmp_path / "relationships.parquet" + frame = _write_parquet(path, rows=12_000) + + def fail_load(self): + raise AssertionError("relationships must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.relationships(path, max_sample_rows=256) + + assert result["available"] is True + assert result["sample"]["source_rows"] == len(frame) + assert result["sample"]["sample_rows"] <= 256 + assert result["sample"]["sampled"] is True + assert result["sample"]["full_materialization"] is False + assert result["sample"]["strategy"] == "streaming_stratified_jitter_global_rows" + assert result["source"]["supports_projection"] is True + edge_pairs = {(edge["source"], edge["target"]) for edge in result["edges"]} + assert ("value", "other") in edge_pairs or ("other", "value") in edge_pairs diff --git a/tests/test_planning.py b/tests/test_planning.py new file mode 100644 index 0000000..a8b2022 --- /dev/null +++ b/tests/test_planning.py @@ -0,0 +1,92 @@ +import json + +import pandas as pd + +import framevitals +from framevitals.cli import main + + +def _dataset() -> pd.DataFrame: + return pd.DataFrame({ + "age": list(range(20, 40)), + "income": [30000 + index * 1000 for index in range(20)], + "city": ["Pune", "Mumbai"] * 10, + "churn": [0, 1] * 10, + }) + + +def test_public_plan_returns_explainable_dict_compatible_plan(): + result = framevitals.plan(_dataset(), mode="quick") + + assert isinstance(result, framevitals.AnalysisPlan) + assert isinstance(result, dict) + assert result["dataset_name"] == "" + assert result.summary()["selected_count"] > 0 + assert any(item["id"] == "structural_profile" for item in result.selected) + assert "FrameVitals Analysis Plan" in result.explain_text() + assert "preview only" in result.explain_text().lower() + + +def test_deep_plan_keeps_target_analysis_and_reserves_modeling_for_research(): + deep = framevitals.plan(_dataset(), mode="deep", target="churn") + deep_ids = {item["id"] for item in deep.selected} + + assert "target_analysis" in deep_ids + assert "baseline_model" not in deep_ids + assert "feature_importance" not in deep_ids + assert deep["config"]["target"] == "churn" + + research = framevitals.plan(_dataset(), mode="research", target="churn") + research_ids = {item["id"] for item in research.selected} + + assert {"target_analysis", "baseline_model", "feature_importance"} <= research_ids + assert research["config"]["target"] == "churn" + + +def test_plan_uses_config_without_writing_artifacts(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + config = tmp_path / "framevitals.toml" + config.write_text( + "[analysis]\nmode = \"deep\"\ntarget = \"churn\"\nartifacts = true\n" + "[resources]\nworkers = 1\n", + encoding="utf-8", + ) + + result = framevitals.plan(_dataset(), config=config) + + assert result["analysis_mode"] == "deep" + assert result["target"] == "churn" + assert result["config"]["artifacts"] is False + assert not (tmp_path / "cleaned").exists() + assert not (tmp_path / "static").exists() + + +def test_cli_plan_can_emit_json(tmp_path, monkeypatch, capsys): + dataset = tmp_path / "dataset.csv" + _dataset().to_csv(dataset, index=False) + output = tmp_path / "plan.json" + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "plan", + str(dataset), + "--mode", + "deep", + "--target", + "churn", + "--format", + "json", + "--output", + str(output), + ], + ) + + assert main() == 0 + stdout_payload = json.loads(capsys.readouterr().out) + file_payload = json.loads(output.read_text(encoding="utf-8")) + + assert stdout_payload["analysis_mode"] == "deep" + assert file_payload["target"] == "churn" + assert file_payload["selection"]["summary"]["selected_count"] > 0 diff --git a/tests/test_planning_boundaries.py b/tests/test_planning_boundaries.py new file mode 100644 index 0000000..79f3ab5 --- /dev/null +++ b/tests/test_planning_boundaries.py @@ -0,0 +1,54 @@ +import subprocess +import sys + +import pandas as pd + +import framevitals as fv + + +def test_plan_exposes_execution_budget(): + frame = pd.DataFrame({ + "x": range(100), + "y": range(100), + }) + + result = fv.plan(frame, mode="standard") + + assert result["execution_budget"]["rows"] == 100 + assert result["execution_budget"]["columns"] == 2 + assert result["selection"]["execution_budget"] == result["execution_budget"] + assert result["source"]["format"] == "pandas" + + +def test_plan_classifies_extreme_shape_policy_without_changing_api(): + # The execution policy itself can reason about extreme shape without ever + # allocating such a dataset. This is validated in adaptive-execution tests; + # here we ensure ordinary plans expose the same policy schema. + result = fv.plan(pd.DataFrame({"x": [1, 2, 3]}), mode="quick") + + budget = result["execution_budget"] + assert budget["scale_class"] in {"normal", "large", "very_large", "extreme"} + assert "max_memory_heavy_parallelism" in budget + assert "bootstrap_sample_rows" in budget + + +def test_plan_does_not_import_full_execution_pipeline(): + code = r''' +import sys +import pandas as pd +import framevitals as fv + +result = fv.plan(pd.DataFrame({"x": range(30), "y": range(30)}), mode="quick") +assert result["analysis_mode"] == "quick" +assert "framevitals.pipeline" not in sys.modules +assert "framevitals.anomaly_ensemble" not in sys.modules +assert "framevitals.deep_statistics_v2" not in sys.modules +''' + completed = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr diff --git a/tests/test_polars_interop.py b/tests/test_polars_interop.py new file mode 100644 index 0000000..ebbcfc8 --- /dev/null +++ b/tests/test_polars_interop.py @@ -0,0 +1,91 @@ +import pytest + +pa = pytest.importorskip("pyarrow") +pl = pytest.importorskip("polars") + +import framevitals as fv +from framevitals.sources import ArrowTableSource, resolve_source + + +def _frame(rows: int = 6_000): + return pl.DataFrame({ + "value": [float(index) for index in range(rows)], + "other": [float(index * 2) for index in range(rows)], + "group": [f"g-{index % 5}" for index in range(rows)], + }) + + +def test_polars_dataframe_uses_generic_arrow_capsule_source(): + frame = _frame(2_000) + + source = resolve_source(frame) + metadata = source.inspect() + + assert isinstance(source, ArrowTableSource) + assert metadata.name == "" + assert metadata.kind == "memory" + assert metadata.format == "arrow" + assert metadata.rows == frame.height + assert metadata.columns == frame.width + assert metadata.supports_projection is True + assert metadata.supports_streaming is True + assert not any( + pa.types.is_string_view(field.type) or pa.types.is_binary_view(field.type) + for field in source.schema() + ) + + info = fv.inspect_source(frame) + assert info == metadata.to_dict() + + +def test_polars_profile_stays_on_arrow_batch_path(monkeypatch): + frame = _frame(12_000) + + def fail_load(self): + raise AssertionError("Polars profiling must not materialize through pandas") + + monkeypatch.setattr(ArrowTableSource, "load", fail_load) + result = fv.profile(frame) + + assert isinstance(result, fv.DiagnosticResult) + assert result.diagnostic == "profile" + assert result.dataset_name == "" + assert result["shape"] == {"rows": frame.height, "columns": frame.width} + assert result["streaming_metadata"]["enabled"] is True + assert result["streaming_metadata"]["full_materialization"] is False + assert result["source_metadata"]["format"] == "arrow" + + +def test_polars_quick_analysis_stays_on_streaming_path(monkeypatch): + frame = _frame(6_000) + + def fail_load(self): + raise AssertionError("Polars analysis must not materialize through pandas") + + monkeypatch.setattr(ArrowTableSource, "load", fail_load) + result = fv.analyze(frame, mode="quick", artifacts=False, workers=1) + + assert result["filename"] == "" + assert result["profile"]["shape"] == { + "rows": frame.height, + "columns": frame.width, + } + assert result["execution"]["streaming"]["enabled"] is True + assert result["execution"]["streaming"]["full_materialization"] is False + + +def test_polars_bounded_statistics_do_not_load_full_dataframe(monkeypatch): + frame = _frame(20_000) + + def fail_load(self): + raise AssertionError("Polars statistics must remain on bounded Arrow batches") + + monkeypatch.setattr(ArrowTableSource, "load", fail_load) + result = fv.statistics(frame, mode="quick", max_pairs=2) + + assert isinstance(result, fv.DiagnosticResult) + assert result.execution["execution_schema_version"] == "1" + assert result.execution["method"] == "bounded_deep_statistics" + assert result.execution["full_materialization"] is False + assert result.execution["source_rows"] == frame.height + assert result.execution["sampled"] is True diff --git a/tests/test_projected_streaming_semantics.py b/tests/test_projected_streaming_semantics.py new file mode 100644 index 0000000..cf515f7 --- /dev/null +++ b/tests/test_projected_streaming_semantics.py @@ -0,0 +1,74 @@ +import pandas as pd +import pytest + +from framevitals.ml_readiness import calculate_ml_readiness_from_profile +from framevitals.streaming_quality import run_streaming_quality_diagnostics + + +def _projected_profile(*, rows: int = 100, source_columns: int = 10_000, profiled_columns: int = 64): + columns = [f"c{i}" for i in range(profiled_columns)] + return { + "shape": {"rows": rows, "columns": source_columns}, + "columns": columns, + "numeric_columns": columns, + "categorical_columns": [], + "missing_counts": {name: rows for name in columns}, + "missing_percent": {name: 100.0 for name in columns}, + "duplicate_rows": 0, + "duplicate_percent": 0.0, + "duplicate_metadata": {"sampled": True}, + "streaming_metadata": { + "enabled": True, + "full_materialization": False, + "sample_rows": min(rows, 50), + "source_columns": source_columns, + "profiled_columns": profiled_columns, + "column_sampled": True, + }, + } + + +def test_ml_readiness_uses_profiled_width_for_projected_missingness(): + profile = _projected_profile() + result = calculate_ml_readiness_from_profile(profile) + + assert result["issues"]["missing_percent"] == pytest.approx(100.0) + assert result["score_scope"] == "full_rows_projected_columns_estimate" + assert result["profiled_columns"] == 64 + assert result["source_columns"] == 10_000 + assert result["execution"]["components"]["missingness"] == ( + "full_rows_projected_columns_exact" + ) + assert result["execution"]["components"]["column_groups"] == "projected_schema_exact" + assert result["execution"]["components"]["duplicate_rate"] == ( + "bounded_row_sample_projected_columns_estimate" + ) + + +def test_streaming_quality_reports_only_columns_actually_checked(): + profile = _projected_profile(rows=200, profiled_columns=64) + sample = pd.DataFrame( + {f"c{i}": [i + row for row in range(10)] for i in range(64)} + ) + + result = run_streaming_quality_diagnostics( + sample, + profile=profile, + source_rows=200, + source_columns=10_000, + max_sample_rows=10, + max_columns=100, + ) + + assert result["columns"] == 10_000 + assert result["profiled_columns"] == 64 + assert result["columns_checked"] == 64 + assert result["truncated_columns"] is True + execution = result["execution"] + assert execution["column_sampled"] is True + assert execution["profile_fact_scope"] == "full_rows_projected_columns" + assert execution["full_source_inputs"] == [] + assert set(execution["projected_full_row_inputs"]) == { + "missingness", + "duplicate_row_estimate", + } diff --git a/tests/test_provenance.py b/tests/test_provenance.py new file mode 100644 index 0000000..fc40f9b --- /dev/null +++ b/tests/test_provenance.py @@ -0,0 +1,116 @@ +from framevitals.provenance import ( + EXECUTION_SCHEMA_VERSION, + execution_provenance, + load_fully_materializes, + normalize_execution, +) +from framevitals.sources import DatasetMetadata + + +def test_execution_provenance_uses_shared_v1_contract(): + source = { + "name": "data.parquet", + "kind": "file", + "format": "parquet", + "rows": 10_000, + "columns": 4, + "size_bytes": 1234, + "materialized": False, + "supports_projection": True, + "supports_streaming": True, + } + + result = execution_provenance( + "bounded_example", + full_materialization=False, + source=source, + sampled=True, + source_rows=10_000, + source_columns=4, + sample_rows=1_000, + strategy="evenly_spaced", + components={"missingness": "exact", "distribution": "sample"}, + reason="Bounded work.", + scope="example", + extra={"pair_budget": 20}, + ) + + assert result["execution_schema_version"] == EXECUTION_SCHEMA_VERSION == "1" + assert result["method"] == "bounded_example" + assert result["full_materialization"] is False + assert result["sampled"] is True + assert result["source_rows"] == 10_000 + assert result["sample_rows"] == 1_000 + assert result["source"] == source + assert result["components"]["missingness"] == "exact" + assert result["pair_budget"] == 20 + + +def test_execution_provenance_omits_unavailable_optional_values(): + result = execution_provenance( + "schema_only", + full_materialization=False, + ) + + assert result == { + "execution_schema_version": "1", + "method": "schema_only", + "full_materialization": False, + } + + +def test_normalize_execution_preserves_legacy_fields_and_adds_contract(): + result = normalize_execution({ + "scope": "bounded_deep_statistics", + "sampled": True, + "sample_rows": 500, + "legacy_flag": "kept", + }) + + assert result["execution_schema_version"] == "1" + assert result["method"] == "bounded_deep_statistics" + assert result["scope"] == "bounded_deep_statistics" + assert result["sampled"] is True + assert result["sample_rows"] == 500 + assert result["legacy_flag"] == "kept" + assert result["full_materialization"] is False + + +def test_materialization_semantics_are_not_file_specific(): + pandas_metadata = DatasetMetadata( + name="", + kind="memory", + format="pandas", + rows=3, + columns=1, + size_bytes=100, + materialized=True, + supports_projection=True, + supports_streaming=False, + ) + arrow_metadata = DatasetMetadata( + name="", + kind="memory", + format="arrow", + rows=3, + columns=1, + size_bytes=24, + materialized=True, + supports_projection=True, + supports_streaming=True, + ) + relation_metadata = DatasetMetadata( + name="", + kind="relation", + format="duckdb", + rows=3, + columns=1, + size_bytes=None, + materialized=False, + supports_projection=True, + supports_streaming=True, + ) + + assert load_fully_materializes(pandas_metadata) is False + assert load_fully_materializes(arrow_metadata) is True + assert load_fully_materializes(relation_metadata) is True diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 4e1b745..b3f8658 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -1,3 +1,5 @@ +from importlib.metadata import version as distribution_version + import pandas as pd import pytest @@ -5,7 +7,7 @@ def test_package_version(): - assert framevitals.__version__ == "0.1.0" + assert framevitals.__version__ == distribution_version("framevitals") def test_analyze_is_public(): diff --git a/tests/test_public_execution_schema.py b/tests/test_public_execution_schema.py new file mode 100644 index 0000000..92aea9f --- /dev/null +++ b/tests/test_public_execution_schema.py @@ -0,0 +1,104 @@ +import numpy as np +import pandas as pd + +import framevitals as fv +from framevitals.sources import DatasetMetadata + + +def _frame(rows: int = 800) -> pd.DataFrame: + values = np.arange(rows, dtype=np.float64) + return pd.DataFrame({ + "value": values, + "other": values * 2.0 + 1.0, + "third": np.sin(values / 10.0), + }) + + +class _LoadedSource: + def __init__(self, frame: pd.DataFrame): + self.frame = frame + + def inspect(self): + return DatasetMetadata( + name="loaded-source", + kind="remote", + format="custom", + rows=len(self.frame), + columns=len(self.frame.columns), + size_bytes=None, + materialized=False, + supports_projection=False, + supports_streaming=False, + ) + + def load(self): + return self.frame.copy() + + +def test_dataframe_statistics_use_execution_schema_v1(): + result = fv.statistics(_frame(), mode="quick", max_pairs=2) + execution = result["execution"] + + assert execution["execution_schema_version"] == "1" + assert execution["method"] == "bounded_deep_statistics" + assert execution["scope"] == "bounded_deep_statistics" + assert execution["full_materialization"] is False + assert execution["source"]["format"] == "pandas" + + +def test_non_pandas_statistics_report_full_materialization(): + result = fv.statistics(_LoadedSource(_frame()), mode="quick", max_pairs=2) + execution = result["execution"] + + assert execution["execution_schema_version"] == "1" + assert execution["method"] == "bounded_deep_statistics" + assert execution["full_materialization"] is True + assert execution["source"]["format"] == "custom" + + +def test_dataframe_anomalies_use_execution_schema_v1(): + result = fv.anomalies( + _frame(), + mode="quick", + max_columns=3, + top_k=5, + ) + execution = result["execution"] + + assert execution["execution_schema_version"] == "1" + assert execution["method"] == "bounded_anomaly_detection" + assert execution["scope"] == "bounded_anomaly_detection" + assert execution["full_materialization"] is False + assert execution["source"]["format"] == "pandas" + + +def test_relationships_use_execution_schema_v1_and_preserve_legacy_sample(): + result = fv.relationships(_frame(), max_sample_rows=64) + execution = result["execution"] + + assert execution["execution_schema_version"] == "1" + assert execution["method"] == "bounded_relationship_graph" + assert execution["full_materialization"] is False + assert execution["sampled"] is True + assert execution["sample_rows"] == 64 + assert execution["source_rows"] == 800 + assert execution["source"]["format"] == "pandas" + + assert result["sample"]["sampled"] is True + assert result["sample"]["sample_rows"] == 64 + assert result["full_materialization"] is False + + +def test_exact_validation_and_gate_use_execution_schema_v1(): + frame = _frame(100) + contract = fv.infer_contract(frame) + + validation = fv.validate(frame, contract) + assert validation["execution"]["execution_schema_version"] == "1" + assert validation["execution"]["method"] == "exact_contract_validation" + assert validation["execution"]["sampled"] is False + + gate = fv.gate(frame, contract=contract) + assert gate["execution"]["execution_schema_version"] == "1" + assert gate["execution"]["method"] == "quality_gate" + assert gate["execution"]["validation"]["execution_schema_version"] == "1" diff --git a/tests/test_public_surface_contract.py b/tests/test_public_surface_contract.py new file mode 100644 index 0000000..e60000d --- /dev/null +++ b/tests/test_public_surface_contract.py @@ -0,0 +1,64 @@ +import framevitals as fv + + +def test_core_public_surface_is_exported(): + expected = { + "AnalysisConfig", + "AnalysisPlan", + "AnalysisResult", + "AnalysisSnapshot", + "SnapshotHistory", + "CleaningPlan", + "ColumnResult", + "DiagnosticResult", + "DataCheck", + "CheckResult", + "DriftResult", + "GateResult", + "ValidationResult", + "inspect_source", + "profile", + "roles", + "health", + "ml_readiness", + "quality", + "statistics", + "anomalies", + "relationships", + "system_info", + "target_analysis", + "analyze", + "plan", + "plan_cleaning", + "clean", + "compare", + "infer_contract", + "validate", + "check", + "run_checks", + "discover_checks", + "gate", + "available_modules", + "create_snapshot", + "load_snapshot", + "compare_snapshots", + "__version__", + } + + assert set(fv.__all__) == expected + for name in expected: + assert hasattr(fv, name), name + + +def test_result_types_remain_dict_compatible(): + for result_type in ( + fv.AnalysisResult, + fv.AnalysisSnapshot, + fv.ColumnResult, + fv.DiagnosticResult, + fv.CheckResult, + fv.DriftResult, + fv.GateResult, + fv.ValidationResult, + ): + assert issubclass(result_type, dict) diff --git a/tests/test_quality_diagnostics_v2.py b/tests/test_quality_diagnostics_v2.py new file mode 100644 index 0000000..e6aee74 --- /dev/null +++ b/tests/test_quality_diagnostics_v2.py @@ -0,0 +1,144 @@ +import numpy as np +import pandas as pd + +import framevitals +from framevitals.column_roles import infer_column_roles +from framevitals.profiler import build_profile +from framevitals.quality_diagnostics import run_quality_diagnostics + + +def _messy_frame() -> pd.DataFrame: + rows = 40 + missing_pattern = [None if index < 10 else index for index in range(rows)] + return pd.DataFrame({ + "customer_id": [f"CUST-{index:03d}" for index in range(rows)], + "order_id": [f"ORD-{index // 2:03d}" for index in range(rows)], + "status": ["active"] * 38 + ["inactive"] * 2, + "base": list(range(rows)), + "base_copy": list(range(rows)), + "numeric_text": [str(index % 20) for index in range(rows)], + "event_text": [f"2026-01-{(index % 20) + 1:02d}" for index in range(rows)], + "city": ["Pune", " pune ", "PUNE", "Mumbai"] * 10, + "notes": ["", "ok", " ", "ready"] * 10, + "ratio": [float(index) for index in range(rows - 1)] + [np.inf], + "mixed": ["1", 2, "3", 4] * 10, + "missing_a": missing_pattern, + "missing_b": list(missing_pattern), + }) + + +def test_quality_diagnostics_detect_real_world_data_problems(): + df = _messy_frame() + profile = build_profile(df) + roles = infer_column_roles(df) + + result = run_quality_diagnostics( + df, + profile=profile, + column_roles=roles, + ) + + assert result["available"] is True + assert result["summary"]["issue_count"] > 0 + + key_columns = {item["column"] for item in result["primary_key_candidates"]} + assert "customer_id" in key_columns + + duplicate_id_columns = { + item["column"] for item in result["identifier_duplicates"] + } + assert "order_id" in duplicate_id_columns + + quasi_columns = {item["column"] for item in result["quasi_constant_columns"]} + assert "status" in quasi_columns + + duplicate_groups = result["duplicate_columns"] + assert any( + item["canonical_column"] == "base" + and "base_copy" in item["duplicate_columns"] + for item in duplicate_groups + ) + + coercions = { + (item["column"], item["suggested_type"]) + for item in result["coercion_candidates"] + } + assert ("numeric_text", "numeric") in coercions + assert ("event_text", "datetime") in coercions + + normalized_columns = { + item["column"] for item in result["category_normalisation"] + } + assert "city" in normalized_columns + + blank_columns = {item["column"] for item in result["blank_strings"]} + assert "notes" in blank_columns + + infinity_columns = {item["column"] for item in result["infinite_values"]} + assert "ratio" in infinity_columns + + mixed_columns = {item["column"] for item in result["mixed_object_types"]} + assert "mixed" in mixed_columns + + pairs = { + tuple(item["columns"]) + for item in result["missingness_relationships"] + } + assert ("missing_a", "missing_b") in pairs + + +def test_quality_diagnostics_are_bounded_and_report_column_truncation(): + df = pd.DataFrame({ + f"col_{index}": list(range(100)) + for index in range(8) + }) + + result = run_quality_diagnostics( + df, + max_sample_rows=20, + max_columns=3, + ) + + assert result["columns_checked"] == 3 + assert result["truncated_columns"] is True + assert result["max_sample_rows"] == 20 + + +def test_public_analysis_surfaces_quality_diagnostics_as_findings(): + result = framevitals.analyze( + _messy_frame(), + mode="quick", + disabled_modules=["cleaning", "ai"], + ) + + diagnostics = result["quality_diagnostics"] + assert diagnostics["available"] is True + assert result["execution"]["module_status"]["quality_diagnostics"] == "ran" + assert "quality_diagnostics" in result["timings_ms"] + + quality_findings = [ + finding + for finding in result.findings + if finding["code"].startswith("quality.") + ] + codes = {finding["code"] for finding in quality_findings} + + assert any(code.startswith("quality.identifier_duplicates.order_id") for code in codes) + assert any(code.startswith("quality.duplicate_columns.base") for code in codes) + assert any(code.startswith("quality.infinite_values.ratio") for code in codes) + assert result.column("city")["quality_findings"] + + +def test_quality_diagnostics_can_be_disabled_without_removing_result_key(): + result = framevitals.analyze( + _messy_frame(), + mode="quick", + disabled_modules=["quality_diagnostics", "cleaning", "ai"], + ) + + assert result["quality_diagnostics"]["skipped"] is True + assert result["execution"]["module_status"]["quality_diagnostics"] == "disabled" + assert not any( + finding["code"].startswith("quality.") + for finding in result.findings + ) diff --git a/tests/test_quality_gates.py b/tests/test_quality_gates.py new file mode 100644 index 0000000..c71733b --- /dev/null +++ b/tests/test_quality_gates.py @@ -0,0 +1,186 @@ +import json + +import pandas as pd + +import framevitals +from framevitals.cli import main +from framevitals.drift_analysis import compare_datasets, severity_at_least + + +def test_identical_datasets_produce_pass_gate(): + frame = pd.DataFrame({ + "value": list(range(30)), + "group": ["A", "B", "C"] * 10, + "event_time": pd.date_range("2026-01-01", periods=30, freq="D"), + }) + + result = compare_datasets(frame, frame.copy()) + + assert result["available"] is True + assert result["gate"]["status"] == "pass" + assert result["summary"]["overall_verdict"] == "stable" + assert result["schema"]["severity"] == "stable" + + +def test_added_schema_column_warns_without_failing(): + reference = pd.DataFrame({"value": list(range(30))}) + current = reference.assign(new_feature=1) + + result = compare_datasets(reference, current) + + assert result["schema"]["added_columns"] == ["new_feature"] + assert result["schema"]["removed_columns"] == [] + assert result["schema"]["severity"] == "moderate" + assert result["gate"]["status"] == "warn" + + +def test_removed_column_or_type_change_is_fail_level(): + reference = pd.DataFrame({ + "value": list(range(30)), + "segment": ["A", "B"] * 15, + }) + current = pd.DataFrame({ + "value": [str(value) for value in range(30)], + }) + + result = compare_datasets(reference, current) + + assert result["schema"]["removed_columns"] == ["segment"] + assert result["schema"]["dtype_changes"][0]["column"] == "value" + assert result["summary"]["overall_verdict"] == "severe" + assert result["gate"]["status"] == "fail" + + +def test_missingness_change_contributes_to_column_severity(): + reference = pd.DataFrame({"value": list(range(50))}) + current_values = list(range(20)) + [None] * 30 + current = pd.DataFrame({"value": current_values}) + + result = compare_datasets(reference, current) + column = result["columns"][0] + + assert column["cur_missing_percent"] == 60.0 + assert column["missingness_delta_percentage_points"] == 60.0 + assert column["missingness_severity"] == "severe" + assert column["drift_severity"] == "severe" + + +def test_categorical_drift_reports_jensen_shannon_distance(): + reference = pd.DataFrame({ + "city": ["Pune"] * 30 + ["Mumbai"] * 20, + }) + current = pd.DataFrame({ + "city": ["Pune"] * 5 + ["Mumbai"] * 5 + ["Delhi"] * 40, + }) + + result = compare_datasets(reference, current) + column = result["columns"][0] + + assert column["jensen_shannon_distance"] is not None + assert column["jensen_shannon_distance"] > 0 + assert "Delhi" in column["new_categories"] + assert column["drift_severity"] in {"moderate", "severe"} + + +def test_datetime_drift_is_analyzed_as_datetime(): + reference = pd.DataFrame({ + "event_time": pd.date_range("2026-01-01", periods=30, freq="D"), + }) + current = pd.DataFrame({ + "event_time": pd.date_range("2026-06-01", periods=30, freq="D"), + }) + + result = compare_datasets(reference, current) + column = result["columns"][0] + + assert column["type"] == "datetime" + assert column["available"] is True + assert column["wasserstein_normalized"] is not None + assert column["ref_min"].startswith("2026-01-01") + assert column["cur_min"].startswith("2026-06-01") + + +def test_requested_column_diagnostics_and_truncation_are_explicit(): + reference = pd.DataFrame({ + "a": list(range(30)), + "b": list(range(30)), + "c": list(range(30)), + }) + current = reference.copy() + + result = compare_datasets( + reference, + current, + columns=["a", "b", "c", "missing"], + max_columns=2, + ) + + assert result["selection"]["truncated"] is True + assert result["selection"]["total_selected_columns"] == 3 + assert result["selection"]["requested_missing_in_reference"] == ["missing"] + assert result["selection"]["requested_missing_in_current"] == ["missing"] + assert result["shared_columns"] == ["a", "b"] + + +def test_severity_threshold_helper_is_ordered(): + assert severity_at_least("severe", "moderate") is True + assert severity_at_least("moderate", "moderate") is True + assert severity_at_least("minor", "moderate") is False + assert severity_at_least("stable", "minor") is False + + +def test_public_quality_results_are_dict_compatible_and_exportable(tmp_path): + reference = pd.DataFrame({ + "value": list(range(30)), + "group": ["A", "B", "C"] * 10, + }) + + drift = framevitals.compare(reference, reference.copy()) + assert isinstance(drift, framevitals.DriftResult) + assert isinstance(drift, dict) + assert drift.status == "pass" + assert "FrameVitals drift" in drift.summary_text() + + drift_path = tmp_path / "drift.json" + drift.to_json(drift_path) + assert json.loads(drift_path.read_text(encoding="utf-8"))["gate"]["status"] == "pass" + + contract = framevitals.infer_contract(reference) + validation = framevitals.validate(reference.copy(), contract) + assert isinstance(validation, framevitals.ValidationResult) + assert isinstance(validation, dict) + assert validation.valid is True + assert validation.status == "pass" + assert "FrameVitals validation" in validation.summary_text() + + +def test_compare_cli_fail_on_is_opt_in(tmp_path, monkeypatch, capsys): + reference = tmp_path / "reference.csv" + current = tmp_path / "current.csv" + pd.DataFrame({"value": list(range(50))}).to_csv(reference, index=False) + pd.DataFrame({"value": list(range(100, 150))}).to_csv(current, index=False) + + monkeypatch.setattr( + "sys.argv", + ["framevitals", "compare", str(reference), str(current)], + ) + assert main() == 0 + capsys.readouterr() + + monkeypatch.setattr( + "sys.argv", + [ + "framevitals", + "compare", + str(reference), + str(current), + "--fail-on", + "moderate", + "--format", + "terminal", + ], + ) + assert main() == 1 + output = capsys.readouterr().out + assert "FrameVitals drift" in output + assert "Severity" in output diff --git a/tests/test_relationship_graph.py b/tests/test_relationship_graph.py new file mode 100644 index 0000000..13d27f0 --- /dev/null +++ b/tests/test_relationship_graph.py @@ -0,0 +1,120 @@ +import numpy as np +import pandas as pd +import pytest + +import framevitals as fv +from framevitals.relationship_graph import build_numeric_relationship_graph + + +def test_relationship_graph_finds_strong_cluster_without_dense_matrix(): + rng = np.random.default_rng(7) + rows = 600 + base = rng.normal(size=rows) + frame = { + "base": base, + "base_copy": base.copy(), + "base_negative": -base, + } + for index in range(57): + frame[f"noise_{index}"] = rng.normal(size=rows) + dataframe = pd.DataFrame(frame) + + result = build_numeric_relationship_graph( + dataframe, + max_sample_rows=256, + min_abs_correlation=0.95, + ) + + assert result["available"] is True + assert result["nodes"] == 60 + assert result["method"] == "bounded_simhash_lsh_then_pearson" + + pairs = { + frozenset((edge["source"], edge["target"])) + for edge in result["edges"] + } + assert frozenset(("base", "base_copy")) in pairs + assert frozenset(("base", "base_negative")) in pairs + + candidate = result["candidate_generation"] + assert candidate["candidate_pairs"] < candidate["total_possible_dense_pairs"] + assert candidate["dense_pairs_avoided"] > 0 + + largest = result["graph"]["components"][0] + assert largest["size"] >= 3 + assert {"base", "base_copy", "base_negative"}.issubset(largest["members"]) + + +def test_relationship_graph_is_row_bounded(): + rng = np.random.default_rng(11) + dataframe = pd.DataFrame( + rng.normal(size=(5_000, 40)), + columns=[f"x_{index}" for index in range(40)], + ) + + result = build_numeric_relationship_graph(dataframe, max_sample_rows=128) + + assert result["sample"]["source_rows"] == 5_000 + assert result["sample"]["sample_rows"] == 128 + assert result["sample"]["sampled"] is True + + +def test_relationship_graph_candidate_budget_is_explicit(): + rng = np.random.default_rng(3) + base = rng.normal(size=200) + dataframe = pd.DataFrame({ + f"copy_{index}": base + rng.normal(scale=1e-8, size=200) + for index in range(100) + }) + + result = build_numeric_relationship_graph( + dataframe, + max_sample_rows=128, + max_candidate_pairs=50, + min_abs_correlation=0.90, + ) + + assert result["candidate_generation"]["candidate_pairs"] <= 50 + assert result["candidate_generation"]["truncated"] is True + assert result["verification"]["verified_relationships"] > 0 + + +def test_relationship_graph_rejects_invalid_controls(): + dataframe = pd.DataFrame({"a": range(30), "b": range(30)}) + + with pytest.raises(ValueError, match="max_sample_rows"): + build_numeric_relationship_graph(dataframe, max_sample_rows=10) + with pytest.raises(ValueError, match="projections"): + build_numeric_relationship_graph(dataframe, projections=17) + with pytest.raises(ValueError, match="min_abs_correlation"): + build_numeric_relationship_graph(dataframe, min_abs_correlation=0) + + +def test_relationship_graph_handles_insufficient_numeric_columns(): + result = build_numeric_relationship_graph( + pd.DataFrame({"label": ["a", "b", "c"]}) + ) + + assert result["available"] is False + assert result["nodes"] == 0 + assert result["edges"] == [] + + +def test_public_relationships_api_preserves_dataset_name(tmp_path): + path = tmp_path / "related.csv" + dataframe = pd.DataFrame({ + "a": np.arange(100, dtype=float), + "b": np.arange(100, dtype=float) * 2, + "noise": np.random.default_rng(9).normal(size=100), + }) + dataframe.to_csv(path, index=False) + + result = fv.relationships(path, min_abs_correlation=0.95) + + assert result["dataset_name"] == "related.csv" + assert result["available"] is True + pairs = { + frozenset((edge["source"], edge["target"])) + for edge in result["edges"] + } + assert frozenset(("a", "b")) in pairs diff --git a/tests/test_release_hardening.py b/tests/test_release_hardening.py new file mode 100644 index 0000000..1e23e67 --- /dev/null +++ b/tests/test_release_hardening.py @@ -0,0 +1,60 @@ +from pathlib import Path +import warnings + +import numpy as np +import pandas as pd +import pytest + +from framevitals import profiler +from framevitals.profiler import build_profile + + +def test_materialized_duplicate_estimate_uses_stratified_jitter(monkeypatch): + frame = pd.DataFrame({"value": [1, 1, 2, 2, 3, 3, 4, 4]}) + seen = {} + + def positions(rows, target_rows): + seen["args"] = (rows, target_rows) + return np.array([0, 2, 4, 6], dtype=np.int64) + + monkeypatch.setattr(profiler, "MAX_EXACT_DUPLICATE_CELLS", 1) + monkeypatch.setattr(profiler, "DUPLICATE_SAMPLE_ROWS", 4) + monkeypatch.setattr(profiler, "_deterministic_stratified_positions", positions) + _, metadata = profiler._duplicate_profile(frame) + assert seen["args"] == (len(frame), 4) + assert metadata["sampled"] is True + assert metadata["strategy"] == "stratified_jitter_global_rows" + + +def test_pandas_profile_excludes_infinities_from_finite_moments_without_hiding_missingness(monkeypatch): + monkeypatch.setenv("FRAMEVITALS_BACKEND", "numpy") + frame = pd.DataFrame( + { + "x": [1.0, 2.0, np.inf, -np.inf, np.nan], + "y": [2.0, 4.0, 8.0, 16.0, 32.0], + } + ) + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + profile = build_profile(frame) + x = profile["numeric_summary"]["x"] + assert x["count"] == 2.0 + assert x["mean"] == pytest.approx(1.5) + assert x["min"] == pytest.approx(1.0) + assert x["max"] == pytest.approx(2.0) + assert profile["missing_counts"]["x"] == 1 + assert profile["numeric_summary_metadata"]["finite_only_moments"] is True + assert profile["correlations"]["x"]["y"] == pytest.approx(1.0) + + +def test_pdf_report_builder_contains_only_framevitals_branding(): + source_path = ( + Path(__file__).resolve().parents[1] + / "src" + / "framevitals" + / "pdf_report_builder.py" + ) + source = source_path.read_text(encoding="utf-8") + assert "DataLens" not in source + assert "DATALENS" not in source + assert "FrameVitals Dataset Report" in source diff --git a/tests/test_result_reporting.py b/tests/test_result_reporting.py new file mode 100644 index 0000000..2acf3a8 --- /dev/null +++ b/tests/test_result_reporting.py @@ -0,0 +1,159 @@ +import json + +import pandas as pd +import pytest + +import framevitals +from framevitals.result import AnalysisResult, ColumnResult + + +def _sample_result() -> AnalysisResult: + return AnalysisResult({ + "dataset_id": "fv_test", + "filename": "customers.csv", + "analysis_mode": "quick", + "artifacts_enabled": False, + "profile": { + "shape": {"rows": 4, "columns": 2}, + "columns": ["age", "city"], + "dtypes": {"age": "float64", "city": "object"}, + "missing_counts": {"age": 1, "city": 0}, + "missing_percent": {"age": 25.0, "city": 0.0}, + "duplicate_rows": 0, + "duplicate_percent": 0.0, + "memory_usage_mb": 0.01, + "numeric_summary": {"age": {"mean": 30.0}}, + "categorical_summary": { + "city": {"unique_values": 2, "top_values": {"Pune": 3}}, + }, + "correlations": {}, + }, + "column_roles": { + "age": { + "roles": ["numeric", "analysis_candidate"], + "unique_count": 3, + "unique_ratio": 0.75, + "non_missing_count": 3, + "is_numeric": True, + "is_categorical": False, + }, + "city": { + "roles": ["categorical", "low_cardinality"], + "unique_count": 2, + "unique_ratio": 0.5, + "non_missing_count": 4, + "is_numeric": False, + "is_categorical": True, + }, + }, + "health": { + "overall_score": 82.5, + "label": "Good", + "details": {"missing_percent": 12.5}, + }, + "ml_readiness": {"score": 72.0, "label": "Mostly Ready"}, + "signals": [ + { + "name": "Data Completeness", + "status": "Review", + "severity": "Medium", + "evidence": "12.5% of dataset cells are missing.", + "recommendation": "Review missing values.", + }, + { + "name": "Temporal Data", + "status": "Detected", + "severity": "Informational", + "evidence": "A date column exists.", + "recommendation": "Use temporal analysis.", + }, + ], + "timings_ms": {"total": 125.0}, + }) + + +def test_analysis_result_remains_dict_compatible(): + result = _sample_result() + + assert isinstance(result, dict) + assert result["health"]["overall_score"] == 82.5 + assert result.get("filename") == "customers.csv" + assert result["result_schema_version"] == "1" + + +def test_findings_are_normalized_from_existing_signals(): + result = _sample_result() + + assert len(result.findings) == 1 + finding = result.findings[0] + assert finding["code"] == "signal.data_completeness" + assert finding["severity"] == "medium" + assert finding["method"] == "signal_engine" + assert result.recommendations == ["Review missing values."] + + +def test_summary_and_column_helpers(): + result = _sample_result() + + summary = result.summary() + assert summary["shape"] == {"rows": 4, "columns": 2} + assert summary["finding_count"] == 1 + assert summary["health"]["overall_score"] == 82.5 + + column = result.column("age") + assert isinstance(column, ColumnResult) + assert column.name == "age" + assert column.missing_percent == 25.0 + assert column.numeric_summary["mean"] == 30.0 + + with pytest.raises(KeyError, match="Column not found"): + result.column("does_not_exist") + + +def test_json_export_is_full_and_round_trips(tmp_path): + result = _sample_result() + + rendered = result.to_json() + payload = json.loads(rendered) + assert payload["profile"]["columns"] == ["age", "city"] + assert payload["findings"][0]["code"] == "signal.data_completeness" + + destination = tmp_path / "nested" / "report.json" + returned = result.to_json(destination) + assert returned == destination + written = json.loads(destination.read_text(encoding="utf-8")) + assert written["dataset_id"] == "fv_test" + + +def test_terminal_and_html_renderers(tmp_path): + result = _sample_result() + + terminal = result.summary_text() + assert "FrameVitals Analysis" in terminal + assert "customers.csv" in terminal + assert "Data Completeness" in terminal + + html = result.to_html() + assert "" in html.lower() + assert "customers.csv" in html + assert "Data Completeness" in html + assert "Inspect raw JSON" in html + + destination = tmp_path / "report.html" + returned = result.to_html(destination) + assert returned == destination + assert destination.exists() + + +def test_public_analyze_returns_analysis_result(): + df = pd.DataFrame({ + "age": [20, 30, 40, 50], + "city": ["Pune", "Mumbai", "Pune", "Nashik"], + }) + + result = framevitals.analyze(df, mode="quick") + + assert isinstance(result, framevitals.AnalysisResult) + assert isinstance(result, dict) + assert result["profile"]["shape"]["rows"] == 4 + assert result.result_schema_version == "1" diff --git a/tests/test_role_keyword_boundaries.py b/tests/test_role_keyword_boundaries.py new file mode 100644 index 0000000..020ac3b --- /dev/null +++ b/tests/test_role_keyword_boundaries.py @@ -0,0 +1,20 @@ +import pandas as pd + +from framevitals.column_roles import infer_column_roles + + +def test_role_keywords_use_token_boundaries(): + frame = pd.DataFrame({ + "paid_amount": [10.0, 20.0, 30.0, 40.0], + "average_score": [0.1, 0.2, 0.3, 0.4], + "customer_id": [101, 102, 103, 104], + "customer_age": [20, 21, 22, 23], + }) + + roles = infer_column_roles(frame) + + assert "id_like" not in roles["paid_amount"]["roles"] + assert "price_like" in roles["paid_amount"]["roles"] + assert "sensitive" not in roles["average_score"]["roles"] + assert "id_like" in roles["customer_id"]["roles"] + assert "sensitive" in roles["customer_age"]["roles"] diff --git a/tests/test_sampling_strategy.py b/tests/test_sampling_strategy.py new file mode 100644 index 0000000..70e6056 --- /dev/null +++ b/tests/test_sampling_strategy.py @@ -0,0 +1,41 @@ +import numpy as np +import pandas as pd + +from framevitals.execution import ( + _deterministic_stratified_positions, + deterministic_sample_frame, +) +from framevitals.streaming_profile import _sample_positions + + +def test_stratified_positions_are_reproducible_sorted_and_cover_full_range(): + first = _deterministic_stratified_positions(10_000, 1_000) + second = _deterministic_stratified_positions(10_000, 1_000) + + assert np.array_equal(first, second) + assert len(first) == 1_000 + assert first[0] == 0 + assert first[-1] == 9_999 + assert np.all(np.diff(first) > 0) + + +def test_stratified_jitter_breaks_periodic_phase_locking(): + rows = 5_000 + period = 51 + frame = pd.DataFrame({"periodic_spike": (np.arange(rows) % period == 0).astype(float)}) + + sampled, metadata = deterministic_sample_frame(frame, 50) + + source_rate = float(frame["periodic_spike"].mean()) + sampled_rate = float(sampled["periodic_spike"].mean()) + assert abs(sampled_rate - source_rate) < 0.15 + assert metadata["strategy"] == "deterministic_stratified_jitter" + assert metadata["sample_rows"] == 50 + assert metadata["seed"] > 0 + + +def test_streaming_and_materialized_paths_share_sampling_positions(): + expected = _deterministic_stratified_positions(20_000, 1_337) + actual = _sample_positions(20_000, 1_337) + + assert np.array_equal(actual, expected) diff --git a/tests/test_scalable_foundations.py b/tests/test_scalable_foundations.py new file mode 100644 index 0000000..22e70e3 --- /dev/null +++ b/tests/test_scalable_foundations.py @@ -0,0 +1,90 @@ +import numpy as np +import pandas as pd + +from framevitals.advanced_indicators import ( + calculate_anomalies, + calculate_freshness, + detect_fairness_review, +) +from framevitals.sources import FileSource, PandasSource, resolve_source + + +def test_advanced_anomaly_scores_match_expected_iqr_density(): + # Keep IQR non-zero so the existing rule is applicable; the final row is an + # obvious outlier in both columns and should therefore have density 1.0. + frame = pd.DataFrame({ + "a": [0, 1, 2, 3, 100], + "b": [1, 2, 3, 4, 50], + }) + + result = calculate_anomalies(frame) + + assert result["anomalous_rows"] == 1 + assert result["highest_score"] == 1.0 + assert result["top_rows"][0]["row_index"] == 4 + assert result["top_rows"][0]["score"] == 1.0 + + +def test_sensitive_name_matching_uses_tokens_not_substrings(): + frame = pd.DataFrame({ + "average_score": [1, 2], + "paid_amount": [5, 6], + "customer_age": [20, 30], + }) + + result = detect_fairness_review(frame) + + assert result["needs_review"] is True + assert result["columns"] == ["customer_age"] + + +def test_freshness_screens_candidates_before_full_parse(): + frame = pd.DataFrame({ + "identifier": [f"item-{i}" for i in range(500)], + "event_date": pd.date_range("2025-01-01", periods=500, freq="D").astype(str), + }) + + result = calculate_freshness(frame) + + assert result["available"] is True + assert result["date_column"] == "event_date" + assert result["oldest_record"] == "2025-01-01" + + +def test_pandas_source_reports_materialized_metadata(): + frame = pd.DataFrame({"x": np.arange(10), "label": ["a"] * 10}) + source = PandasSource(frame) + + metadata = source.inspect() + + assert metadata.rows == 10 + assert metadata.columns == 2 + assert metadata.materialized is True + assert metadata.supports_projection is True + assert source.load().equals(frame) + assert source.load() is not frame + + +def test_file_source_reports_cheap_metadata_without_loading(tmp_path): + path = tmp_path / "data.csv" + path.write_text("x,y\n1,2\n3,4\n", encoding="utf-8") + source = resolve_source(path) + + assert isinstance(source, FileSource) + metadata = source.inspect() + assert metadata.name == "data.csv" + assert metadata.format == "csv" + assert metadata.rows is None + assert metadata.size_bytes == path.stat().st_size + + loaded = source.load() + assert loaded.shape == (2, 2) + + +def test_resolve_source_rejects_unsupported_objects(): + try: + resolve_source(object()) + except TypeError as exc: + assert "DatasetSource" in str(exc) + else: + raise AssertionError("resolve_source should reject unsupported objects") diff --git a/tests/test_semantic_types.py b/tests/test_semantic_types.py new file mode 100644 index 0000000..fafabcc --- /dev/null +++ b/tests/test_semantic_types.py @@ -0,0 +1,128 @@ +import pandas as pd + +import framevitals +from framevitals.column_roles import infer_column_roles, summarize_roles +from framevitals.dataset_signals import detect_dataset_signals +from framevitals.profiler import build_profile +from framevitals.semantic_types import infer_semantic_types + + +def test_semantic_type_detector_recognizes_common_value_patterns(): + cases = { + "email": pd.Series(["a@example.com", "b@example.com", "c@example.org"]), + "url": pd.Series(["https://example.com/a", "http://openai.com", "www.python.org"]), + "uuid": pd.Series([ + "550e8400-e29b-41d4-a716-446655440000", + "6ba7b810-9dad-11d1-80b4-00c04fd430c8", + "6ba7b811-9dad-11d1-80b4-00c04fd430c8", + ]), + "ip_address": pd.Series(["192.168.1.1", "10.0.0.8", "2001:db8::1"]), + "phone": pd.Series(["+91 98765 43210", "020-1234-5678", "+1 (415) 555-0100"]), + "percentage": pd.Series(["12%", "3.5%", "-0.5%"]), + "currency": pd.Series(["₹1,200", "$19.99", "INR 5000"]), + "json": pd.Series(['{"a": 1}', '[1, 2]', '{"ok": true}']), + "boolean_token": pd.Series(["yes", "NO", "true", "off"]), + } + + for expected, series in cases.items(): + result = infer_semantic_types(series) + assert result["primary"] == expected + assert result["candidates"][0]["confidence"] >= 0.7 + + +def test_semantic_type_detector_is_bounded_and_ignores_numeric_columns(): + values = [f"user{index}@example.com" for index in range(250)] + text_result = infer_semantic_types(pd.Series(values), max_samples=40) + numeric_result = infer_semantic_types(pd.Series([1, 2, 3, 4])) + + assert text_result["primary"] == "email" + assert text_result["sample_size"] == 40 + assert numeric_result == { + "primary": None, + "candidates": [], + "sample_size": 0, + } + + +def test_value_semantics_augment_column_roles_without_name_hints(): + df = pd.DataFrame({ + "contact_value": [ + "a@example.com", + "b@example.com", + "c@example.com", + "d@example.com", + ], + "record_key": [ + "550e8400-e29b-41d4-a716-446655440000", + "6ba7b810-9dad-11d1-80b4-00c04fd430c8", + "6ba7b811-9dad-11d1-80b4-00c04fd430c8", + "6ba7b812-9dad-11d1-80b4-00c04fd430c8", + ], + "web": [ + "https://example.com/a", + "https://example.com/b", + "https://example.com/c", + "https://example.com/d", + ], + }) + + roles = infer_column_roles(df) + + assert roles["contact_value"]["semantic_type"] == "email" + assert "email_like" in roles["contact_value"]["roles"] + assert "sensitive" in roles["contact_value"]["roles"] + + assert roles["record_key"]["semantic_type"] == "uuid" + assert "uuid_like" in roles["record_key"]["roles"] + assert "id_like" in roles["record_key"]["roles"] + + assert roles["web"]["semantic_type"] == "url" + assert "url_like" in roles["web"]["roles"] + + summary = summarize_roles(roles) + assert summary["email_like"] == ["contact_value"] + assert summary["uuid_like"] == ["record_key"] + assert summary["url_like"] == ["web"] + + +def test_dataset_signals_reuse_semantic_roles(): + df = pd.DataFrame({ + "contact": ["a@example.com", "b@example.com", "c@example.com"], + "homepage": ["https://a.example", "https://b.example", "https://c.example"], + "host": ["10.0.0.1", "10.0.0.2", "10.0.0.3"], + "price_text": ["₹100", "₹200", "₹300"], + }) + profile = build_profile(df) + roles = infer_column_roles(df) + + signals = detect_dataset_signals(df, profile, column_roles=roles) + + assert signals["has_email_like_columns"] is True + assert signals["email_like_columns"] == ["contact"] + assert signals["has_url_like_columns"] is True + assert signals["url_like_columns"] == ["homepage"] + assert signals["has_ip_address_like_columns"] is True + assert signals["ip_address_like_columns"] == ["host"] + assert signals["has_currency_like_columns"] is True + assert signals["currency_like_columns"] == ["price_text"] + assert set(signals["sensitive_columns"]) >= {"contact", "host"} + + +def test_column_result_surfaces_semantic_information(): + df = pd.DataFrame({ + "contact": [ + "a@example.com", + "b@example.com", + "c@example.com", + "d@example.com", + ], + "value": [1, 2, 3, 4], + }) + + result = framevitals.analyze(df, mode="quick") + column = result.column("contact") + + assert column.semantic_type == "email" + assert column.semantic_sample_size == 4 + assert column.semantic_candidates[0]["type"] == "email" + assert "email_like" in column.roles diff --git a/tests/test_snapshots.py b/tests/test_snapshots.py new file mode 100644 index 0000000..84e58f9 --- /dev/null +++ b/tests/test_snapshots.py @@ -0,0 +1,147 @@ +import json + +import pytest + +from framevitals.result import AnalysisResult +from framevitals.snapshots import ( + SnapshotHistory, + compare_snapshots, + create_snapshot, + load_snapshot, +) + + +def _result(*, columns=None, health=80.0, missing=None, findings=None): + columns = columns or {"age": "int64", "city": "object"} + missing = missing or {name: 0.0 for name in columns} + return AnalysisResult({ + "dataset_id": "fv_runtime_id", + "filename": "customers.csv", + "analysis_mode": "quick", + "profile": { + "shape": {"rows": 10, "columns": len(columns)}, + "columns": list(columns), + "dtypes": columns, + "missing_percent": missing, + "duplicate_percent": 0.0, + "memory_usage_mb": 0.1, + }, + "health": { + "overall_score": health, + "label": "Good", + "components": {"completeness": health}, + }, + "ml_readiness": { + "score": 75.0, + "label": "Mostly Ready", + "issues": {"missing_percent": max(missing.values(), default=0.0)}, + }, + "signals": [], + "findings": findings or [], + "config": { + "mode": "quick", + "target": None, + "artifacts": False, + "workers": 2, + }, + }) + + +def test_snapshot_is_compact_deterministic_state(tmp_path): + first = _result().snapshot() + second = _result().snapshot() + + assert first["snapshot_schema_version"] == "1" + assert first["fingerprint"] == second["fingerprint"] + assert "profile" not in first + assert first["state"]["dataset"]["dtypes"]["age"] == "int64" + + path = tmp_path / "baseline.json" + returned = _result().snapshot(path) + assert path.exists() + assert returned["fingerprint"] == load_snapshot(path)["fingerprint"] + + +def test_snapshot_diff_reports_schema_health_missingness_and_findings(): + reference = _result( + health=85.0, + findings=[{"code": "signal.duplicate_records"}], + ).snapshot() + current = _result( + columns={"age": "float64", "city": "object", "segment": "object"}, + health=76.5, + missing={"age": 10.0, "city": 0.0, "segment": 5.0}, + findings=[{"code": "signal.data_completeness"}], + ).snapshot() + + diff = compare_snapshots(reference, current) + + assert diff["changed"] is True + assert diff["schema"]["added_columns"] == ["segment"] + assert diff["schema"]["type_changes"]["age"] == { + "reference": "int64", + "current": "float64", + } + assert diff["missingness_changes"]["age"]["delta"] == 10.0 + assert diff["health_delta"] == -8.5 + assert diff["findings"]["new"] == ["signal.data_completeness"] + assert diff["findings"]["resolved"] == ["signal.duplicate_records"] + + +def test_snapshot_json_roundtrip(tmp_path): + path = tmp_path / "snapshot.json" + snapshot = _result().snapshot(path) + + payload = json.loads(path.read_text(encoding="utf-8")) + assert payload["fingerprint"] == snapshot["fingerprint"] + loaded = load_snapshot(path) + assert loaded.diff(snapshot)["changed"] is False + + +def test_snapshot_history_persists_orders_and_compares_latest(tmp_path): + history = SnapshotHistory(tmp_path / "history") + baseline = create_snapshot(_result(health=90.0)) + baseline["created_at"] = "2026-08-14T08:00:00+00:00" + current = create_snapshot( + _result( + health=72.5, + findings=[{"code": "quality.high_missingness.age"}], + ) + ) + current["created_at"] = "2026-08-15T08:00:00+00:00" + + first_path = history.add(baseline, label="baseline release") + second_path = history.add(current, label="production") + + assert len(history) == 2 + assert "baseline-release" in first_path.name + assert "production" in second_path.name + assert history.previous()["fingerprint"] == baseline["fingerprint"] + assert history.latest()["fingerprint"] == current["fingerprint"] + + diff = history.compare_latest() + assert diff["changed"] is True + assert diff["health_delta"] == -17.5 + assert diff["findings"]["new"] == ["quality.high_missingness.age"] + + timeline = history.timeline() + assert [row["health_score"] for row in timeline] == [90.0, 72.5] + assert [row["finding_count"] for row in timeline] == [0, 1] + assert all(row["filename"] == "customers.csv" for row in timeline) + + +def test_snapshot_history_can_add_analysis_result_directly(tmp_path): + history = SnapshotHistory(tmp_path / "history") + path = history.add(_result(health=88.0), label="nightly") + + assert path.exists() + assert len(history) == 1 + assert history.latest()["state"]["health"]["overall_score"] == 88.0 + + +def test_snapshot_history_requires_two_entries_for_latest_diff(tmp_path): + history = SnapshotHistory(tmp_path / "history") + history.add(_result()) + + with pytest.raises(ValueError, match="at least two"): + history.compare_latest() diff --git a/tests/test_source_inspection.py b/tests/test_source_inspection.py new file mode 100644 index 0000000..e9d076b --- /dev/null +++ b/tests/test_source_inspection.py @@ -0,0 +1,80 @@ +import pandas as pd +import pytest + +import framevitals as fv +from framevitals.sources import DatasetMetadata, DelimitedTextSource + + +def test_inspect_source_reports_pandas_capabilities(): + frame = pd.DataFrame({ + "value": [1, 2, 3], + "label": ["a", "b", "c"], + }) + + info = fv.inspect_source(frame) + + assert info["name"] == "" + assert info["kind"] == "memory" + assert info["format"] == "pandas" + assert info["rows"] == 3 + assert info["columns"] == 2 + assert info["size_bytes"] > 0 + assert info["materialized"] is True + assert info["supports_projection"] is True + assert info["supports_streaming"] is False + + +def test_inspect_source_reports_csv_fallback_without_arrow(tmp_path, monkeypatch): + path = tmp_path / "data.csv" + pd.DataFrame({"value": [1, 2, 3]}).to_csv(path, index=False) + monkeypatch.setattr(DelimitedTextSource, "_pyarrow_csv", lambda self: None) + + info = fv.inspect_source(path) + + assert info["name"] == "data.csv" + assert info["kind"] == "file" + assert info["format"] == "csv" + assert info["rows"] is None + assert info["columns"] is None + assert info["size_bytes"] == path.stat().st_size + assert info["materialized"] is False + assert info["supports_projection"] is False + assert info["supports_streaming"] is False + + +def test_inspect_source_accepts_custom_dataset_source(): + class CustomSource: + def inspect(self): + return DatasetMetadata( + name="custom", + kind="remote", + format="custom", + rows=42, + columns=4, + size_bytes=None, + materialized=False, + supports_projection=True, + supports_streaming=False, + ) + + def load(self): + return pd.DataFrame({"value": [1]}) + + info = fv.inspect_source(CustomSource()) + + assert info == { + "name": "custom", + "kind": "remote", + "format": "custom", + "rows": 42, + "columns": 4, + "size_bytes": None, + "materialized": False, + "supports_projection": True, + "supports_streaming": False, + } + + +def test_inspect_source_rejects_unsupported_input(): + with pytest.raises(TypeError, match="dataset path"): + fv.inspect_source(object()) diff --git a/tests/test_statistics_streaming.py b/tests/test_statistics_streaming.py new file mode 100644 index 0000000..4b1e906 --- /dev/null +++ b/tests/test_statistics_streaming.py @@ -0,0 +1,50 @@ +import numpy as np +import pandas as pd +import pytest + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + +import framevitals +from framevitals.sources import ParquetSource + + +def _write_statistics_parquet(path, rows: int = 12_000) -> pd.DataFrame: + frame = pd.DataFrame({ + "value": np.linspace(1.0, 100.0, rows), + "other": np.linspace(3.0, 203.0, rows), + "group": [f"g-{index % 5}" for index in range(rows)], + "event_time": pd.date_range("2026-01-01", periods=rows, freq="min"), + }) + pq.write_table( + pa.Table.from_pandas(frame, preserve_index=False), + path, + row_group_size=777, + ) + return frame + + +def test_public_statistics_streams_bounded_parquet_sample(tmp_path, monkeypatch): + path = tmp_path / "statistics.parquet" + frame = _write_statistics_parquet(path) + + def fail_load(self): + raise AssertionError("statistics must not materialize the complete Parquet file") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.statistics(path, mode="quick", max_pairs=2) + + assert result["dataset_name"] == "statistics.parquet" + execution = result["execution"] + assert execution["execution_schema_version"] == "1" + assert execution["method"] == "bounded_deep_statistics" + assert execution["scope"] == "bounded_deep_statistics" + assert execution["full_materialization"] is False + assert execution["source_rows"] == len(frame) + assert execution["source_columns"] == len(frame.columns) + assert execution["sample_rows"] == 1_000 + assert execution["sampled"] is True + assert execution["strategy"] == "streaming_stratified_jitter_global_rows" + assert execution["pair_budget"] == 2 + assert execution["source"]["format"] == "parquet" + assert result["source"]["supports_streaming"] is True diff --git a/tests/test_stream_change.py b/tests/test_stream_change.py new file mode 100644 index 0000000..d39c054 --- /dev/null +++ b/tests/test_stream_change.py @@ -0,0 +1,96 @@ +import numpy as np +import pandas as pd + +from framevitals.budgeted_analysis import run_budgeted_time_series +from framevitals.execution import derive_execution_budget +from framevitals.stream_change import PageHinkleyMeanShift, scan_ordered_mean_shift + + +def test_page_hinkley_detects_sustained_batch_mean_shift(): + rng = np.random.default_rng(42) + detector = PageHinkleyMeanShift(threshold=8.0, min_updates=8) + + for value in rng.normal(0.0, 0.08, size=24): + detector.update(float(value)) + assert detector.detected is False + + for value in rng.normal(2.5, 0.08, size=12): + detector.update(float(value)) + if detector.detected: + break + + snapshot = detector.snapshot() + assert snapshot["detected"] is True + assert snapshot["direction"] == "up" + assert snapshot["detected_at_batch"] is not None + assert snapshot["sufficient_batches"] is True + + +def test_page_hinkley_stays_quiet_on_stationary_batch_means(): + rng = np.random.default_rng(7) + detector = PageHinkleyMeanShift(threshold=10.0, min_updates=8) + + for value in rng.normal(10.0, 0.15, size=80): + detector.update(float(value)) + + assert detector.detected is False + assert detector.snapshot()["updates"] == 80 + + +def test_page_hinkley_ignores_nonfinite_observations(): + detector = PageHinkleyMeanShift() + detector.update(None) + detector.update(float("nan")) + detector.update(float("inf")) + + assert detector.count == 0 + assert detector.snapshot()["sufficient_batches"] is False + + +def test_ordered_mean_shift_scan_detects_series_regime_change(): + rng = np.random.default_rng(123) + series = pd.Series(np.concatenate([ + rng.normal(0.0, 0.1, size=600), + rng.normal(3.0, 0.1, size=600), + ])) + + result = scan_ordered_mean_shift(series, windows=24) + + assert result["available"] is True + assert result["detected"] is True + assert result["direction"] == "up" + assert result["windows"] == 24 + + +def test_budgeted_time_series_attaches_mean_shift_when_series_is_detected(monkeypatch): + rows = 1_200 + rng = np.random.default_rng(5) + values = np.concatenate([ + rng.normal(0.0, 0.1, size=600), + rng.normal(2.5, 0.1, size=600), + ]) + frame = pd.DataFrame({ + "event_time": pd.date_range("2025-01-01", periods=rows, freq="h"), + "value": values, + }) + + def fake_time_series(work, target_column=None): + return { + "available": True, + "detected_date_column": "event_time", + "numeric_column": "value", + } + + monkeypatch.setattr( + "framevitals.budgeted_analysis.detect_and_analyze_time_series", + fake_time_series, + ) + budget = derive_execution_budget(rows, 2, mode="standard") + result = run_budgeted_time_series(frame, budget=budget) + + assert result["mean_shift"]["available"] is True + assert result["mean_shift"]["detected"] is True + assert result["execution"]["mean_shift_detection_enabled"] is True + assert result["execution"]["method"] == "bounded_time_series" + assert result["execution"]["scope"] == "bounded_time_series" + assert result["execution"]["adaptive_strategy"] == "ordered_page_hinkley_mean_shift" diff --git a/tests/test_streaming_bounded_pipeline.py b/tests/test_streaming_bounded_pipeline.py new file mode 100644 index 0000000..04c0514 --- /dev/null +++ b/tests/test_streaming_bounded_pipeline.py @@ -0,0 +1,81 @@ +import pandas as pd +import pytest + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + +import framevitals +import framevitals.analysis_api as analysis_api +import framevitals.pipeline as materialized_pipeline + + +def test_streaming_analysis_does_not_reenter_materialized_core(tmp_path, monkeypatch): + path = tmp_path / "bounded-only.parquet" + frame = pd.DataFrame({ + "x": range(2_000), + "y": [value * 2 for value in range(2_000)], + "group": [f"g-{value % 5}" for value in range(2_000)], + }) + pq.write_table(pa.Table.from_pandas(frame, preserve_index=False), path, row_group_size=333) + + def fail(*args, **kwargs): + raise AssertionError("streaming analysis re-entered materialized core work") + + # A streaming file must not fall back to the materialized public dispatcher. + monkeypatch.setattr(analysis_api, "run_full_analysis", fail) + # Nor may the bounded scheduler recompute core pandas profile/role/health/quality state. + monkeypatch.setattr(materialized_pipeline, "build_profile", fail) + monkeypatch.setattr(materialized_pipeline, "infer_column_roles", fail) + monkeypatch.setattr(materialized_pipeline, "calculate_health_score", fail) + monkeypatch.setattr(materialized_pipeline, "calculate_ml_readiness", fail) + monkeypatch.setattr(materialized_pipeline, "run_quality_diagnostics", fail) + + result = framevitals.analyze(path, mode="quick", artifacts=False, workers=2) + + scheduler = result["execution"]["bounded_scheduler"] + assert scheduler["enabled"] is True + assert scheduler["core_reprofiled"] is False + assert scheduler["source_budget_scope"] == "source_shape" + assert scheduler["parallelism_budget_scope"] == "bounded_sample" + assert result["profile"]["shape"] == {"rows": 2_000, "columns": 3} + assert result["execution"]["streaming"]["full_materialization"] is False + assert "profile" not in result["timings_ms"] + assert result["timings_ms"]["streaming_profile"] > 0 + + +def test_streaming_bounded_scheduler_uses_source_budget_for_ultra_wide_limits(tmp_path): + # The focused unit check is shape-only: the source budget must retain the + # ultra-wide relationship cap even though the retained sample is narrower. + from framevitals.execution import derive_execution_budget + from framevitals.streaming_bounded_pipeline import run_streaming_bounded_modules + + sample = pd.DataFrame({f"c{i}": range(100) for i in range(32)}) + source_budget = derive_execution_budget(100_000, 10_000, mode="standard") + payload = run_streaming_bounded_modules( + sample, + dataset_id="test", + original_filename="", + analysis_mode="standard", + target_column=None, + parallel_workers=4, + source_budget=source_budget, + column_roles={}, + skip_ai=True, + disabled_modules={ + "deep_statistics", + "anomaly_detection", + "time_series", + "text_profile", + "modeling", + "explainability", + "cleaning", + "charts", + "quality_diagnostics", + }, + ) + + budget = payload["execution"]["budget"] + assert budget["rows"] == 100_000 + assert budget["columns"] == 10_000 + assert budget["relationship_pair_budget"] == 10 + assert payload["execution"]["bounded_scheduler"]["sample_columns"] == 32 diff --git a/tests/test_streaming_drift.py b/tests/test_streaming_drift.py new file mode 100644 index 0000000..7453d9e --- /dev/null +++ b/tests/test_streaming_drift.py @@ -0,0 +1,82 @@ +import numpy as np +import pandas as pd +import pytest + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + +import framevitals +from framevitals.sources import ParquetSource + + +def _write_drift_parquet(path, *, rows: int, shift: float = 0.0) -> pd.DataFrame: + frame = pd.DataFrame({ + "value": np.arange(rows, dtype=np.float64) + shift, + "segment": [f"s-{index % 5}" for index in range(rows)], + }) + frame.loc[::211, "value"] = np.nan + pq.write_table( + pa.Table.from_pandas(frame, preserve_index=False), + path, + row_group_size=777, + ) + return frame + + +def test_compare_streams_large_parquet_and_reports_true_source_shapes( + tmp_path, + monkeypatch, +): + reference_path = tmp_path / "reference.parquet" + current_path = tmp_path / "current.parquet" + reference = _write_drift_parquet(reference_path, rows=60_000) + current = _write_drift_parquet(current_path, rows=72_000, shift=250.0) + + def fail_load(self): + raise AssertionError("drift comparison must not fully materialize Parquet inputs") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.compare(reference_path, current_path) + + assert result["available"] is True + assert result["reference_name"] == "reference.parquet" + assert result["current_name"] == "current.parquet" + assert result["ref_shape"] == [len(reference), len(reference.columns)] + assert result["cur_shape"] == [len(current), len(current.columns)] + assert result["row_count_change_percent"] == pytest.approx(20.0) + + execution = result["execution"] + assert execution["method"] == "bounded_source_compare" + assert execution["full_materialization"] is False + assert execution["sample_limit_rows_per_source"] == 50_000 + assert execution["reference"]["source_rows"] == len(reference) + assert execution["reference"]["sample_rows"] == 50_000 + assert execution["reference"]["sampled"] is True + assert execution["current"]["source_rows"] == len(current) + assert execution["current"]["sample_rows"] == 50_000 + assert execution["current"]["sampled"] is True + assert execution["components"]["source_shape"] == "exact" + assert execution["components"]["value_distributions"] == "bounded_row_sample" + assert execution["components"]["missingness"] == "bounded_row_sample" + + +def test_reference_only_gate_reuses_streaming_drift_path(tmp_path, monkeypatch): + reference_path = tmp_path / "gate-reference.parquet" + current_path = tmp_path / "gate-current.parquet" + _write_drift_parquet(reference_path, rows=12_000) + _write_drift_parquet(current_path, rows=12_000, shift=25.0) + + def fail_load(self): + raise AssertionError("reference-only gate must not materialize Parquet inputs") + + monkeypatch.setattr(ParquetSource, "load", fail_load) + result = framevitals.gate(current_path, reference=reference_path) + + assert result["checks_run"] == ["drift"] + assert result["execution"]["validation"] is None + drift_execution = result["execution"]["drift"] + assert drift_execution["full_materialization"] is False + assert drift_execution["reference"]["source_rows"] == 12_000 + assert drift_execution["current"]["source_rows"] == 12_000 + assert drift_execution["reference"]["strategy"] == "full_stream_via_batches" + assert drift_execution["current"]["strategy"] == "full_stream_via_batches" diff --git a/tests/test_streaming_exact_reuse.py b/tests/test_streaming_exact_reuse.py new file mode 100644 index 0000000..a1822b4 --- /dev/null +++ b/tests/test_streaming_exact_reuse.py @@ -0,0 +1,92 @@ +from framevitals.streaming_exact_reuse import reuse_streaming_exact_statistics + + +def test_reuses_full_stream_moments_without_overwriting_sample_diagnostics(): + payload = { + "profile": { + "numeric_summary": { + "x": { + "count": 100, + "mean": 12.5, + "std": 3.25, + "min": -2.0, + "25%": 10.0, + "50%": 12.0, + "75%": 15.0, + "max": 30.0, + "skewness": 1.25, + "kurtosis": 2.0, + } + }, + "numeric_summary_metadata": { + "backend": "rust", + "method": "native_streaming_accumulator", + "higher_moments": "full_stream_exact", + }, + "streaming_metadata": { + "enabled": True, + "numeric_backend": "rust", + }, + }, + "deep_statistics_v2": { + "numeric_statistics": { + "x": { + "count": 20, + "mean": 99.0, + "std": 77.0, + "min": 1.0, + "max": 200.0, + "median": 13.0, + "skewness": 0.4, + "skewness_label": "Approximately Symmetric", + "kurtosis": -2.0, + "kurtosis_label": "Light-tailed", + "distribution_fit": {"available": True, "best_fit": {"name": "norm"}}, + } + }, + "execution": {"scope": "bounded_deep_statistics"}, + }, + } + + result = reuse_streaming_exact_statistics(payload) + stats = result["deep_statistics_v2"]["numeric_statistics"]["x"] + + assert stats["count"] == 100 + assert stats["mean"] == 12.5 + assert stats["std"] == 3.25 + assert stats["min"] == -2.0 + assert stats["max"] == 30.0 + assert stats["median"] == 13.0 + assert stats["skewness"] == 1.25 + assert stats["skewness_label"] == "Highly Skewed" + assert stats["kurtosis"] == 2.0 + assert stats["kurtosis_label"] == "Heavy-tailed" + assert stats["distribution_fit"]["best_fit"]["name"] == "norm" + assert stats["summary_provenance"]["backend"] == "rust" + assert set(stats["summary_provenance"]["reused_exact_fields"]) == { + "count", + "mean", + "std", + "min", + "max", + "skewness", + "kurtosis", + } + assert "skewness" not in stats["summary_provenance"]["sample_derived_fields"] + assert "kurtosis" not in stats["summary_provenance"]["sample_derived_fields"] + reuse = result["deep_statistics_v2"]["execution"]["exact_once_reuse"] + assert reuse["enabled"] is True + assert reuse["columns_reused"] == 1 + assert set(reuse["fields"]) >= {"skewness", "kurtosis"} + + +def test_noop_when_deep_statistics_are_not_applicable(): + payload = { + "profile": { + "numeric_summary": {"x": {"count": 10, "mean": 2.0}}, + "streaming_metadata": {"enabled": True, "numeric_backend": "rust"}, + }, + "deep_statistics_v2": None, + } + + assert reuse_streaming_exact_statistics(payload) is payload diff --git a/tests/test_streaming_sketches.py b/tests/test_streaming_sketches.py new file mode 100644 index 0000000..320a461 --- /dev/null +++ b/tests/test_streaming_sketches.py @@ -0,0 +1,44 @@ +import numpy as np +import pytest + +from framevitals.streaming_sketches import ( + NumpyLogQuantileSketch, + PYTHON_NUMERIC_SKETCH_CELL_BUDGET, + should_use_full_stream_numpy_sketch, +) + + +def test_numpy_log_quantile_sketch_is_mergeable_and_bounded(): + values = np.arange(1, 10_001, dtype=np.float64) + left = NumpyLogQuantileSketch().update(values[:5_000]) + right = NumpyLogQuantileSketch().update(values[5_000:]) + merged = left.merge(right) + + assert merged.count == 10_000 + assert merged.quantile(0.5) == pytest.approx(5_000, rel=0.03) + assert merged.quantile(0.95) == pytest.approx(9_500, rel=0.03) + assert merged.bin_count < 1_000 + snapshot = merged.snapshot() + assert snapshot["method"] == "numpy_log_quantile_sketch" + assert snapshot["relative_accuracy"] == 0.01 + + +def test_numpy_log_quantile_sketch_handles_signed_zero_and_nonfinite_values(): + sketch = NumpyLogQuantileSketch().update( + np.array([-100.0, -10.0, -0.0, 0.0, 10.0, 100.0, np.nan, np.inf]) + ) + + assert sketch.count == 6 + assert sketch.zero_count == 2 + assert sketch.quantile(0.0) < 0 + assert sketch.quantile(1.0) > 0 + + +def test_numpy_stream_sketch_budget_uses_full_stream_for_normal_numeric_workloads(): + assert should_use_full_stream_numpy_sketch(100_000, 100) is True + assert 100_000 * 100 <= PYTHON_NUMERIC_SKETCH_CELL_BUDGET + + +def test_numpy_stream_sketch_budget_avoids_ultra_wide_cpu_regression(): + assert should_use_full_stream_numpy_sketch(100_000, 6_750) is False + assert should_use_full_stream_numpy_sketch(100_000, 10_000) is False diff --git a/tests/test_target_intelligence_v2.py b/tests/test_target_intelligence_v2.py new file mode 100644 index 0000000..54d36d6 --- /dev/null +++ b/tests/test_target_intelligence_v2.py @@ -0,0 +1,122 @@ +import pandas as pd + +import framevitals +from framevitals.column_roles import infer_column_roles +from framevitals.target_intelligence import run_target_intelligence + + +def test_classification_target_intelligence_ranks_mixed_feature_types(): + target = [0, 1] * 20 + df = pd.DataFrame({ + "numeric_signal": [value * 10 + (index % 3) for index, value in enumerate(target)], + "segment": ["stay" if value == 0 else "leave" for value in target], + "noise": list(range(40)), + "churn": target, + }) + + result = run_target_intelligence( + df, + target_column="churn", + column_roles=infer_column_roles(df), + ) + + assert result["available"] is True + assert result["task_type"] == "classification" + assert result["split_guidance"]["strategy"] == "stratified_random_split" + + associations = {item["feature"]: item for item in result["top_associations"]} + assert associations["numeric_signal"]["method"] == "point_biserial" + assert associations["numeric_signal"]["score"] > 0.9 + assert associations["segment"]["method"] == "cramers_v" + assert associations["segment"]["score"] > 0.9 + + leakage_features = {item["feature"] for item in result["leakage"]["warnings"]} + assert "segment" in leakage_features + + +def test_regression_target_intelligence_uses_spearman_and_correlation_ratio(): + target = list(range(1, 41)) + df = pd.DataFrame({ + "linear": [value * 3 for value in target], + "bucket": ["low"] * 20 + ["high"] * 20, + "target": target, + }) + + result = run_target_intelligence(df, target_column="target") + associations = {item["feature"]: item for item in result["top_associations"]} + + assert result["task_type"] == "regression" + assert associations["linear"]["method"] == "spearman" + assert associations["linear"]["score"] == 1.0 + assert associations["bucket"]["method"] == "correlation_ratio" + assert associations["bucket"]["score"] > 0.8 + + +def test_time_like_columns_trigger_split_review_guidance(): + df = pd.DataFrame({ + "event_date": pd.date_range("2026-01-01", periods=30, freq="D"), + "value": list(range(30)), + "target": [0, 1] * 15, + }) + + result = run_target_intelligence(df, target_column="target") + + assert result["split_guidance"]["strategy"] == "review_time_aware_split" + assert "event_date" in result["split_guidance"]["time_candidates"] + + +def test_target_intelligence_warns_for_identifier_like_target(): + df = pd.DataFrame({ + "feature": list(range(20)), + "customer_id": [f"CUST-{index:04d}" for index in range(20)], + }) + + result = run_target_intelligence(df, target_column="customer_id") + warning_codes = {item["code"] for item in result["warnings"]} + + assert "target.id_like" in warning_codes + + +def test_public_analyze_includes_target_intelligence_even_in_quick_mode(): + df = pd.DataFrame({ + "age": list(range(20, 40)), + "plan": ["basic", "pro"] * 10, + "churn": [0, 1] * 10, + }) + + result = framevitals.analyze(df, target="churn", mode="quick") + + intelligence = result["target_intelligence"] + assert intelligence["available"] is True + assert intelligence["target_column"] == "churn" + assert intelligence["task_type"] == "classification" + assert isinstance(intelligence["top_associations"], list) + assert "target_intelligence" in result["timings_ms"] + + +def test_target_leakage_warning_is_normalized_into_public_findings(): + target = [0, 1] * 10 + df = pd.DataFrame({ + "leaked_target": target, + "age": list(range(20, 40)), + "churn": target, + }) + + result = framevitals.analyze(df, target="churn", mode="quick") + target_findings = [ + finding + for finding in result.findings + if finding["method"] == "target_intelligence" + ] + + assert any( + finding["code"] == "target.leakage.leaked_target" + for finding in target_findings + ) + leakage_finding = next( + finding + for finding in target_findings + if finding["code"] == "target.leakage.leaked_target" + ) + assert leakage_finding["severity"] == "critical" + assert "prediction time" in leakage_finding["recommendation"] diff --git a/tests/test_visualization_engine.py b/tests/test_visualization_engine.py index af2ee4a..ff1de27 100644 --- a/tests/test_visualization_engine.py +++ b/tests/test_visualization_engine.py @@ -1,16 +1,13 @@ import pandas as pd +import pytest -import framevitals.visualizer as visualizer_module +pytest.importorskip("matplotlib") +pytest.importorskip("seaborn") -from framevitals.chart_planner import ( - build_chart_plan, -) -from framevitals.column_roles import ( - infer_column_roles, -) -from framevitals.profiler import ( - build_profile, -) +import framevitals.visualizer as visualizer_module +from framevitals.chart_planner import build_chart_plan +from framevitals.column_roles import infer_column_roles +from framevitals.profiler import build_profile def make_dataset(): diff --git a/tests/test_web_dependency_boundaries.py b/tests/test_web_dependency_boundaries.py new file mode 100644 index 0000000..32f13f2 --- /dev/null +++ b/tests/test_web_dependency_boundaries.py @@ -0,0 +1,37 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("flask") + + +def test_web_app_does_not_eagerly_import_optional_ai_or_report_stack(): + repo_root = Path(__file__).resolve().parents[1] + env = os.environ.copy() + env["PYTHONPATH"] = str(repo_root / "src") + completed = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; import app; " + "assert 'framevitals.ai_agent' not in sys.modules; " + "assert 'pydantic' not in sys.modules; " + "assert 'framevitals.report_generator' not in sys.modules; " + "assert 'framevitals.pdf_report_builder' not in sys.modules; " + "assert 'framevitals.visualizer' not in sys.modules; " + "assert 'framevitals.explainability' not in sys.modules; " + "assert 'matplotlib' not in sys.modules; " + "assert 'seaborn' not in sys.modules" + ), + ], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr