diff --git a/.codex/skills/crpto/SKILL.md b/.codex/skills/crpto/SKILL.md index 2620b44..c03cfe4 100644 --- a/.codex/skills/crpto/SKILL.md +++ b/.codex/skills/crpto/SKILL.md @@ -1,13 +1,10 @@ # CRPTO Skill -Use this skill when working inside `C:\Users\carlos\Documents\Paper_CRPTO`. -CRPTO is a standalone academic paper project for Conformal Robust -Predict-Then-Optimize in credit risk. The repository is public-facing, but the -paper champion artifacts are frozen. +Use this skill inside `C:\Users\carlos\Documents\Paper_CRPTO`. CRPTO is a +single-author IJDS paper and reproducibility bundle, not a production service. +Prefer simple code, frozen evidence, and one coherent manuscript claim. -## Required Context - -Before structural work, read these files in order: +## Read First 1. `docs/ACADEMIC_CONTEXT.md` 2. `docs/SCOPE_AND_GOVERNANCE.md` @@ -15,60 +12,48 @@ Before structural work, read these files in order: 4. `CONTRIBUTING.md` 5. `EXTRACTION_MANIFEST.md` 6. `configs/crpto_publication_targets.yaml` -7. `docs/research/README.md` - -Use the local project context over generic habits. This is a single-author, -static-dataset, no-production academic repo. Keep code simple, functional, and -close to the existing style. - -## Platform - -- Work Windows-first in PowerShell. -- Prefer `uv run ...` for Python, Quarto, dbt, DVC, MLflow, pytest, and ruff. -- The local venv is `.venv/Scripts/python.exe`. -- Do not introduce Unix-only shell assumptions. - -## Champion Rules - -The current IJDS paper-facing body claim is the promoted pool93 finite-grid -frontier closure. It is a deterministic policy-grid re-evaluation over the -frozen upstream PD/calibration/conformal artifacts, not a retraining run. - -- Terminal run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` -- Body point source run: - `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` -- Policy family: `claim_micro_ext_body_cap345` -- Policy mode: `capped_blended_uncertainty` -- Robust return: `$184,832.48` -- Return-floor surplus: `$14,367.94` -- `V(alpha=0.01)`: `0.035350` -- `Gamma_CP(alpha=0.01)`: `0.162616` -- Endpoint budget upper at `alpha=0.01`: `0.24508374` -- Markov cap at `alpha=0.01`: `0.34508374` -- Exact alpha violation: `0.0` -- Declared alpha-grid pass: `8/8` -- Consolidated frontier: `50,010` deduplicated semantic policies, `27,508` - eligible all-alpha above-floor policies. -- Terminal exact search: `37,068/37,068` all-alpha passers and `296,544` - completed exact candidate-alpha checks. - -The frozen upstream baseline remains retained for provenance and as the -declared return floor: - -- Run tag: `ijds-rebaseline-2026-06-07` -- Policy: `bound_aware_276k_economic_champion` -- Robust return: `$170,464.54` -- `V(alpha=0.01)`: `0.028875` -- `Gamma_CP(alpha=0.01)`: `0.187987` -- Exact pass: `True` -- Former robust region: `45/45` - -The older run tag `paper-thesis-final-economic-2026-04-06` is historical -provenance only. Do not use it as the active body claim. - -Never overwrite these frozen artifacts unless the user explicitly asks for a -champion rebuild: + +Use Windows PowerShell and `uv run`. Do not introduce Unix-only workflow +assumptions. + +## Active IJDS Policy + +- Run tag: + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` +- Exact conformal replay: target `alpha=0.10`, frozen used alpha `0.095`. +- Decision score: `q=(p+u)/2`. +- Risk tolerance: `tau=0.17`. +- Objective: point-PD expected net return; conformal `q` is the risk guardrail. +- Selector: nine round-number policies on November 2017; five satisfy full + budget, effective-PD, and deterministic `B_u<=0.28` screens. Outcomes are + stored separately from the 12-column selector frame. +- Audit: an outcome-free December replay selects the same policy; opening + outcomes afterward gives weighted default `0.145650` and miscoverage + `0.124925`, so stability is not reported as selected-set validity. +- Full OOT: 276,869 candidates, 308 funded, `$179,327.59` realized return, + `0.039375` weighted default, `0.036875` weighted miscoverage. +- `Gamma_CP=0.176102`, `Gamma_residual=0.088051`, endpoint `0.258051`. +- Markov sensitivity: threshold `0.574279` with probability bound `0.316228` + under weighted funded-set validity. +- Matched point-PD: `$196,369.14`, `0.118400` weighted default, endpoint + `0.921317`, threshold `1.237545`. + +The exact paper-facing source is: + +`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` + +A35--A40 provide exact-alpha, selector, temporal, grade, bootstrap, and matched +comparison evidence. The final selector does not read OOT outcomes, but prior +project development inspected the static OOT corpus. Say "retrospective +lockbox replay," not "preregistered," "prospective," or "untouched holdout." + +Do not revive these as active claims: approximate alpha-0.01 scaling, `8/8`, +the 50,010-policy frontier, `0.345084`, capped/tail-selected policies, or OOT +outcome-selected hyperparameters. + +## Frozen Provenance + +Never overwrite the manifest-protected upstream or historical pool93 files: - `models/pd_canonical.cbm` - `models/pd_canonical_calibrator.pkl` @@ -76,116 +61,65 @@ champion rebuild: - `models/conformal_policy_status.json` - `data/processed/conformal_intervals_mondrian.parquet` - `data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/` -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` -- `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv` -- `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` -- `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` -- `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` +- historical `crpto_tableA35..A40_pool93_*` files +- historical pool93 governance JSON files - `EXTRACTION_MANIFEST.json` -Protected DVC stages: - -- Do not run without explicit permission: `crpto.portfolio.bound_exact_eval` - and any Optuna/HPO or policy search. -- Treat as protected revalidation stages: `crpto.pd.champion`, - `crpto.conformal.intervals`, `crpto.conformal.validation`, and - `crpto.portfolio.optimization`. If the user explicitly permits them, run them - as a drift check and validate hashes before/after. -- Safe paper stages: `crpto.paper.export_tables`, `crpto.paper.evidence`, - `crpto.paper.journal_package`, `crpto.paper.tail_satisficing_audit`, - `crpto.paper.figures`, `crpto.paper.spo_stability`, and `crpto.book.render`. - -If the user says a change may touch the champion, isolate the work: - -- Use a dedicated branch and a clear run label. -- Write new outputs under a new path; do not replace frozen champion files. -- Produce a drift report with expected tolerances. -- Run `just validate-champion` before and after. -- Do not update `EXTRACTION_MANIFEST.json` unless the user explicitly approves - a new frozen release. - -## Current Journal Scope - -The active paper scope is the IJDS pool93 certificate plus bounded diagnostics: - -- A19/Fig15: regret-auditability frontier in the main narrative. -- A35: promoted pool93 finite-grid return-bound frontier. -- A36: funded-set grade composition audit for the selected pool93 allocation. -- A37: pool93 selected-allocation LGD/CVaR/OCE tail-risk repricing. -- A38: pool93 selected-allocation cluster-bound sensitivity; Markov remains - the body theorem because the cluster thresholds are not tighter here. -- A39: fixed-allocation bootstrap diagnostic; it does not resample model, - solver, conformal intervals or policy search. -- A20--A22: legacy tail-risk/OCE/CVaR diagnostic package retained in the - supplement, not as the promoted pool93 selector. -- A23--A24: multi-distribution/online coverage diagnostics, not universal - conditional-coverage claims. -- A25--A34: Prosper/Freddie external economic replication, not new Lending Club - champions. - -Do not re-run champion search, HPO, conformal interval generation or protected -portfolio stages to support this pack. Use existing artifacts unless the user -explicitly asks for a new isolated experiment with a claim target, evidence -gate, artifact sink and stop rule. - -## Objective Experiments - -OCE/CVaR/satisficing can be implemented as an isolated research objective or -scoring layer. The default rule is: - -- It may read existing allocations, predictions, intervals, or shortlist - artifacts. -- It may generate new diagnostic tables, figures, configs, tests, and docs. -- It must not replace the champion objective, rank-1 policy, or frozen outputs. -- A tail-satisficing challenger audit may read or re-score frozen allocations - under a new paper/audit stage if outputs are new and the status marks - `champion_promotion_changed=false`. It must not replace the A35 pool93 body - selector without a new promotion protocol. -- If used for a new search, store results under a new experiment path and make - the comparison explicit against the active pool93 body claim and the frozen - upstream rebaseline. - -## Submission Closeout - -For the current submission, keep these gates visible: - -- Consolidate A19/Fig15, A20--A39, paper/supplement, docs, manifest hashes and - `dvc.lock`. -- Sweep the manuscript for stale numbers, captions, body-vs-appendix placement, - and IJDS length. -- Convert the final `.qmd` into the official IJDS LaTeX template when the PDF - is ready for submission. -- Keep GitHub/DagsHub/MLflow disclosure compatible with double-anonymous - review: anonymize in the manuscript, reveal in cover letter or after review - according to venue policy. -- Create the reproducible release tag/bundle only after the final PDF passes. -- Treat `dvc push` as optional and credential-dependent. - -## Standard Checks - -Use focused checks while editing, then close with the strongest feasible set: +New experiments must use a distinct run tag and write under +`data/processed/experiments/champion_reopen/` and +`models/experiments/champion_reopen/`. Never replace frozen paths. + +Protected DVC stages are `crpto.pd.champion`, +`crpto.conformal.intervals`, `crpto.conformal.validation`, +`crpto.portfolio.optimization`, and `crpto.portfolio.bound_exact_eval`. +Experiments may read their outputs. Run a protected stage only with explicit +permission and a drift report. + +## Method Boundary + +The submitted method has one linear policy. Capped, tail, OCE/CVaR, SPO+, +multi-distribution, online, causal, and external-data variants are comparators +or diagnostics, not additional CRPTO methods. + +Keep these distinctions explicit: + +- exact conformal quantile replay versus approximate width scaling; +- point PD in the economic objective versus conformal `q` in the constraint; +- deterministic `weighted outcome <= B_u + V` versus the + assumption-conditional Markov statement; +- calibration-only final ranking versus historical OOT-aware development; +- full-OOT averages versus temporal heterogeneity; +- retrospective contrasts versus causal or universal dominance. + +## Paper Workflow + +Safe paper work may regenerate active A35--A40, figures, Quarto outputs, and the +official IJDS PDF. Keep body, supplement, submission TeX, governance JSON, and +claim-sync tests numerically aligned. + +Standard closeout: ```powershell -uv run python scripts/build_crpto_journal_package.py -uv run pytest tests/test_publication_targets.py -q -uv run pytest tests/test_scripts/test_build_crpto_journal_package.py -q -uv run pytest tests/test_pool93_body_claim_sync.py -q -just smoke +uv run python scripts/build_ijds_calibration_selected_evidence.py +uv run pytest tests/test_ijds_active_claim_sync.py -q just lint +just type-check +just type-advisory-full +just smoke just validate-champion just paper-submission +just paper-submission-official uv run dvc status --no-updates ``` -For a full milestone, also run `just test`, `just type-check`, and the relevant -Quarto render. Do not bypass hooks. +Run `just drift-gate` after changes to conformal or PD semantics. Do not bypass +hooks, commit secrets, or alter `EXTRACTION_MANIFEST.json`. -## Writing And Docs +## Writing -- Spanish for book, paper prose, and research notes. -- English for code, identifiers, CI, and docstrings. -- Do not reformat the whole book in one pass. -- Avoid broad refactors unless they directly reduce risk for the current task. -- Never commit secrets, `.env`, local credentials, or heavyweight raw data. +- Paper and code identifiers are English; project notes may be Spanish. +- Lead with data, method, decision, and managerial implication. +- Report the price of robustness and temporal failures as prominently as wins. +- Treat reproducibility as evidence quality, not as the sole novelty. +- Keep the main IJDS body within 25 pages; move proofs and diagnostics to the + separate supplement. diff --git a/.github/workflows/tests-full.yml b/.github/workflows/tests-full.yml index 5ad9622..5b3f528 100644 --- a/.github/workflows/tests-full.yml +++ b/.github/workflows/tests-full.yml @@ -72,6 +72,7 @@ jobs: tests/test_features/ \ tests/test_crpto_final_sync.py \ tests/test_quarto_book_guardrails.py \ + tests/test_publication_integrity.py \ -q - name: Upload coverage if generated diff --git a/.gitignore b/.gitignore index 2e428de..109697f 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,8 @@ models/feature_artifacts_runtime_status.json # Reports (large generated outputs) reports/run_logs/ +reports/api-docs/ +reports/ci/ reports/notebook_images/ reports/paper_material/ reports/crpto/experiments/ @@ -149,6 +151,9 @@ models/experiments/** !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_frontier.json +!models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json +!models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json +!models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json configs/experiments/ !configs/experiments/ configs/experiments/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8434421..78d3195 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,15 +47,15 @@ repos: stages: [pre-push] - id: pytest-smoke - name: pytest smoke (final-sync + quarto guardrails) - entry: uv run pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py -q + name: just smoke (publication guardrails) + entry: just smoke language: system pass_filenames: false stages: [pre-push] - id: validate-champion name: validate champion artefacts - entry: uv run python -c "import json; from pathlib import Path; m=json.loads(Path('EXTRACTION_MANIFEST.json').read_text()); print('manifest OK, keys:', len(m))" + entry: just validate-champion language: system pass_filenames: false stages: [pre-push] diff --git a/AGENTS.md b/AGENTS.md index 9311ff3..6bdc9f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,8 +10,8 @@ Lectura obligatoria, en este orden: 1. [`CLAUDE.md`](CLAUDE.md) — reglas de operación, champion congelado (esquema dual-tag pool93), stages prohibidos, comandos y convenciones. 2. [`docs/research/active_claims_2026-07-04.md`](docs/research/active_claims_2026-07-04.md) - — registro vigente de claims del paper (body point pool93, denominadores - finite-grid, reopen gate). + — registro vigente de claims del paper (replay exacto al 90%, selector de + calibración 3x3, política lineal 50/50 y reopen gate). 3. [`.codex/skills/crpto/SKILL.md`](.codex/skills/crpto/SKILL.md) — skill de Codex con el detalle operativo del certificado y sus artefactos. diff --git a/CLAUDE.md b/CLAUDE.md index b58a842..219e7db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,22 +33,32 @@ El "champion congelado" se refiere al **pipeline de búsqueda** que produjo las ## Champion congelado — NO RE-CORRER -Esquema dual-tag (detalle en `docs/SCOPE_AND_GOVERNANCE.md`). Pool93 es una -re-evaluación determinista de una grilla finita de políticas sobre los mismos -intervalos conformal congelados; **no regenera ningún artefacto upstream**. +El modelo PD, calibrador, intervalos y bundle pool93 del manifest permanecen +congelados. El body IJDS usa un replay exacto y una política nueva bajo un run +tag aislado; **no regenera ni sobreescribe ningún artefacto upstream**. -**Body claim del paper IJDS (pool93, activo):** +**Body claim del paper IJDS (activo):** | Campo | Valor | | --- | --- | -| Run tag | `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` | -| Policy mode | `capped_blended_uncertainty` (familia `claim_micro_ext_body_cap345`) | -| Retorno robusto | `$184,832.48` | -| V(α=0.01) | `0.035350` | -| Γ_CP(α=0.01) | `0.162616` | -| Markov cap (α=0.01) | `0.345084` | -| Alpha grid | `8/8`, violación exacta `0.0` | -| Evidencia | A35–A39 + JSONs de gobernanza pool93 | +| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` | +| Conformal | exact replay at target `alpha=0.10` (frozen used alpha `0.095`) | +| Policy | `q=(p+u)/2`, `tau=0.17`, point-PD economic objective | +| Selector | 9 policies on Nov 2017; deterministic `B_u<=0.28`; 5 eligible; same policy in outcome-free Dec replay | +| Realized return | `$179,327.59` on a `$1M` budget | +| Weighted default / miscoverage | `0.039375 / 0.036875` | +| Gamma_CP / Gamma_residual | `0.176102 / 0.088051` | +| Endpoint / Markov threshold | `0.258051 / 0.574279` | +| Matched point-PD A40 | return cost `8.678%`; default reduction `7.9025` pp; threshold reduction `66.3266` pp | +| Evidence | exact alpha A35 + split selector/audit A36 + temporal/funded-set/baseline A37--A40 | + +The exact policy-facing quantities come from +`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. +The primary claim is the simple calibration-selected guardrail, deterministic +endpoint screen, independent December audit, and exact funded-set accounting. +December miscoverage `0.124925` documents that stable selection is not +selected-set validity. Markov remains an assumption-conditional sensitivity, +not a selector or headline novelty. **Cadena upstream congelada (histórica; su retorno es el return floor declarado del pool93):** @@ -62,7 +72,8 @@ intervalos conformal congelados; **no regenera ningún artefacto upstream**. | Exact pass | `True` | | Región robusta | `45/45` | -Artefactos congelados cuyos hashes están en `EXTRACTION_MANIFEST.json` y **no se tocan** sin permiso: +Artefactos históricos congelados cuyos hashes están en +`EXTRACTION_MANIFEST.json` y **no se tocan** sin permiso: - `models/pd_canonical.cbm` - `models/pd_canonical_calibrator.pkl` @@ -70,12 +81,16 @@ Artefactos congelados cuyos hashes están en `EXTRACTION_MANIFEST.json` y **no s - `models/conformal_policy_status.json` - `data/processed/conformal_intervals_mondrian.parquet` - `data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/` -- `reports/crpto/tables/crpto_tableA35..A39_pool93_*.csv/.tex` (evidencia pool93) +- `reports/crpto/tables/crpto_tableA35..A40_pool93_*.csv/.tex` (evidencia pool93) - `models/experiments/champion_reopen/...__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` -- `models/experiments/champion_reopen/...__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` +- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` - `EXTRACTION_MANIFEST.json` -La sincronía del body claim con el paper la vigila `tests/test_pool93_body_claim_sync.py`. +La sincronía del body claim activo con el paper la vigila +`tests/test_ijds_active_claim_sync.py`. `tests/test_pool93_body_claim_sync.py` +queda limitado a la integridad de procedencia histórica. Stages DVC que regeneran estos artefactos (`crpto.pd.champion`, `crpto.conformal.intervals`, `crpto.conformal.validation`, `crpto.portfolio.optimization`, `crpto.portfolio.bound_exact_eval`) **no se ejecutan** sin permiso. Validar con `crpto-validate-champion` antes de cualquier merge. diff --git a/EXTRACTION_MANIFEST.json b/EXTRACTION_MANIFEST.json index a3f66d0..13c1f30 100644 --- a/EXTRACTION_MANIFEST.json +++ b/EXTRACTION_MANIFEST.json @@ -1,5 +1,5 @@ { - "schema_version": 5, + "schema_version": 6, "project_name": "CRPTO", "source_project": "local source repository; read-only during extraction; absolute source path intentionally omitted in final package", "source_project_note": "Formal IJDS rebaseline computed in the standalone Paper_CRPTO repository; the parent WSL project is provenance only.", @@ -893,9 +893,9 @@ "hash_source": "feature_config_pickle_retirement_2026-06-13" }, "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv": { - "sha256": "fbaf05312426058a6b7691f696198666ee289374305a3867dd84f589bddb0132", - "bytes": 2264, - "hash_source": "pool93_ijds_promotion_2026-07-02" + "sha256": "819b2927856ffcc4b6993aa0f9e35839a562803cd7685ddb3013390e64659a44", + "bytes": 2334, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" }, "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv": { "sha256": "9d02382c22c7bd58fdbb287de43d0ca09714bae1753b4d30eb1609cb1502740f", @@ -918,9 +918,9 @@ "hash_source": "pool93_ijds_promotion_2026-07-02" }, "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex": { - "sha256": "37b4c4d0d2da6f4dbcae55031c07905a12875776463eecae6cab086dc5e8d6f9", - "bytes": 956, - "hash_source": "pool93_ijds_promotion_2026-07-02" + "sha256": "b951567302b3547573d1fbf569ef1ede3e15f8b2e1484b5d2a13399eff4eedf9", + "bytes": 1233, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" }, "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex": { "sha256": "20ab4514684f1d2d2cb888be022385d0db771d9fd78b4c87ebff19958e96ad9d", @@ -951,6 +951,31 @@ "sha256": "e686f8b0f76c8b3a63351b97ea3761f55fb8921a1c23b051e59904163e8a79de", "bytes": 7996, "hash_source": "pool93_ijds_promotion_2026-07-02" + }, + "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv": { + "sha256": "25a9420250f275ccf927d23de6a2a5cbfc46bf411552f99fb684fc2cc21ddafe", + "bytes": 492, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" + }, + "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex": { + "sha256": "4ab75f6f6c106947d4be50d568f390e7c53bf47f01117cfdf04bfbcc9bd1f4de", + "bytes": 328, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" + }, + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json": { + "sha256": "5d3f82559ae60e450af9e30644fe08cb728f0fe39b0fa7b05b50dc29668be6ed", + "bytes": 15290, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" + }, + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json": { + "sha256": "b698e4269b2dfbd028840a432bc3b926cc7f6dd9ec63b8550c392b38dc445c48", + "bytes": 9823, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" + }, + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json": { + "sha256": "bd1dfd75fdc5a13334ab71040de1750eabda05898660e93c8ab1723e8dbaa597", + "bytes": 3810, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" } }, "validation_results": { @@ -4318,9 +4343,10 @@ ] }, "pool93_ijds_promotion": { - "promoted_at_utc": "2026-07-02T06:05:17.323703+00:00", - "note": "Pool93 finite-grid body claim promoted for the IJDS paper. The champion_metrics block above remains the frozen ijds-rebaseline-2026-06-07 upstream chain (PD champion, Mondrian conformal intervals, bound-aware 276k search); pool93 is a deterministic policy-grid re-evaluation over the same frozen conformal intervals and does not regenerate any upstream artifact.", + "promoted_at_utc": "2026-07-09T23:10:36+00:00", + "note": "Pool93 policy-aware certificate semantics v2 promoted for the IJDS paper. The selected allocation, upstream PD/calibration/conformal chain, alpha grid and finite-grid denominators are unchanged. Exact endpoint budgets are rehydrated from existing evaluations; A40 adds a matched point-PD baseline without rerunning protected stages.", "terminal_run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "certificate_semantics_run_tag": "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2", "consolidated_run_tags": [ "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", @@ -4341,10 +4367,16 @@ "realized_total_return": 184832.475845, "return_floor_surplus": 14367.935845, "alpha01_gamma_cp": 0.162616, + "alpha01_gamma_internalized": 0.089032, + "alpha01_gamma_residual": 0.073584, "alpha01_weighted_miscoverage_V": 0.03535, - "endpoint_budget_upper_alpha01": 0.24508374, - "markov_cap_alpha01": 0.34508374, + "alpha01_realized_risk_tolerance_excess": 0.0, + "endpoint_budget_alpha01": 0.245084, + "endpoint_budget_upper_alpha01": 0.245084, + "markov_threshold_alpha01": 0.345084, + "markov_cap_alpha01": 0.345084, "alpha_grid_pass": "8/8", + "alpha01_n_funded": 314, "n_funded_mean": 320.5, "declared_return_floor": 170464.54, "bootstrap_return_interval_p025_p975": [ @@ -4364,19 +4396,49 @@ "n_all_alpha_passers": 37068, "n_all_alpha_passers_above_return_floor": 14814 }, + "certificate_semantics_audit": { + "status": "corrected_from_existing_exact_bound_evaluations", + "body_selection_unchanged": true, + "materially_changed_policies": 10423, + "materially_understated_policies": 2866, + "maximum_legacy_understatement": 0.2413235, + "legacy_under_0_50_excluded_by_exact_threshold": 716, + "affected_policy_modes": [ + "segment_relative_tail_blended_uncertainty", + "tail_blended_uncertainty" + ] + }, + "matched_point_pd_baseline": { + "candidate_universe": 276869, + "budget": 1000000.0, + "risk_tolerance": 0.1715, + "point_pd_realized_return": 196369.14000000004, + "selected_crpto_realized_return": 184832.47584455396, + "realized_return_cost": 11536.66415544608, + "realized_return_cost_pct": 5.874988379256577, + "weighted_default_rate_reduction": 0.08305000000000001, + "weighted_miscoverage_reduction": 0.08305000000000001, + "markov_threshold_reduction": 0.4354954303914803, + "claim_boundary": "Frozen OOT matched-policy audit; no causal, prospective or universal dominance claim." + }, "protected_artifacts": [ "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv", "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", "reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv", "reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv", "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv", + "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv", "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex", "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex", "reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.tex", "reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.tex", "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.tex", + "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex", "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json", - "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json" + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json", + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json", + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json", + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json" ] } } diff --git a/EXTRACTION_MANIFEST.md b/EXTRACTION_MANIFEST.md index 9cf475d..d67cfb1 100644 --- a/EXTRACTION_MANIFEST.md +++ b/EXTRACTION_MANIFEST.md @@ -8,16 +8,20 @@ exists, and how `tests/test_manifest_regression.py` enforces it. ## TL;DR -- **Schema version**: 5 (top-level key `schema_version`). -- **Dual-tag governance**: - - frozen upstream baseline: `ijds-rebaseline-2026-06-07`; - - active IJDS body claim: pool93 finite-grid frontier closure - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal`. -- **182 critical files** are hashed under `critical_hashes` (SHA256 + byte +- **Schema version**: 6 (top-level key `schema_version`). +- **Manifest scope**: frozen upstream baseline plus the historical pool93 + promotion recorded at extraction time. The current IJDS manuscript policy is + newer and is governed separately by + `docs/research/active_claims_2026-07-04.md`. +- **187 critical files** are hashed under `critical_hashes` (SHA256 + byte count). -- **Pool93 body claim**: return `$184,832.48`, `V(alpha=0.01)=0.035350`, - `Gamma_CP(alpha=0.01)=0.162616`, Markov cap `0.345084`, exact alpha - violation `0.0`, declared alpha-grid pass `8/8`. +- **Historical pool93 claim**: return `$184,832.48`, `V(alpha=0.01)=0.035350`, + `Gamma_CP=0.162616`, `Gamma_res=0.073584`, endpoint `0.245084`, exact + Markov loss threshold `0.345084`, realized risk-tolerance excess `0.0`, and + declared alpha-grid pass `8/8`. +- **Matched A40 baseline**: CRPTO pays `5.875%` realized return relative to a + point-PD LP and reduces weighted default/miscoverage by `8.305` percentage + points under matched operating constraints. - **6 files are flagged as non-overwriteable** without a fresh run tag: - `models/pd_canonical.cbm` - `models/pd_canonical_calibrator.pkl` @@ -46,7 +50,7 @@ exists, and how `tests/test_manifest_regression.py` enforces it. | `generated_at_utc` | When the manifest was produced. | | `summary` | Free-text human description of the extraction scope. | | `champion_metrics` | Frozen upstream baseline numbers retained as provenance and as the declared return floor. | -| `pool93_ijds_promotion` | Active IJDS body-claim metadata for the pool93 finite-grid frontier closure. | +| `pool93_ijds_promotion` | Historical IJDS metadata frozen at extraction time; retained as provenance, not the current manuscript claim. | | `critical_hashes` | Map `relative_path → {sha256, bytes, hash_source}` for every file the paper depends on. | | `validation_results` | Output of the extraction-time guardrail tests. | | `files` | Inventory of files copied/created during extraction. | diff --git a/README.md b/README.md index 9bc7f02..4c61475 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,24 @@ Pipeline de investigación y libro Quarto que acompañan el paper **CRPTO**, una > CRPTO opera como repositorio standalone: GitHub, DVC y MLflow apuntan a recursos propios del paper. La historia de extracción y aprendizajes queda documentada en [`docs/PROJECT_HISTORY.md`](docs/PROJECT_HISTORY.md). -## Champion congelado +## Claim IJDS activo | Campo | Valor | | --- | --- | -| Run tag | `ijds-rebaseline-2026-06-07` | -| Policy | `bound_aware_276k_economic_champion` | -| Retorno robusto | **$170,464.54** | -| `V(α=0.01)` | `0.028875` | -| `Γ_CP(α=0.01)` | `0.187987` | -| `α=0.01 exact pass` | `True` | -| Región robusta | `45/45` | +| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` | +| Conformal | replay exacto al `90%` (`alpha=0.10`, used `0.095`) | +| Política | `q=(p+u)/2`, `tau=0.17`; PD puntual en el objetivo y `q` en el guardrail | +| Selector | grilla `3x3` en noviembre; cap determinista `B_u<=0.28`; misma política en auditoría de diciembre | +| Retorno realizado | **$179,327.59** | +| Default / miscoverage ponderados | `0.039375 / 0.036875` | +| `Gamma_CP / Gamma_residual` | `0.176102 / 0.088051` | +| Endpoint / contabilidad observada / umbral condicional | `0.258051 / 0.294926 / 0.574279` | +| Baseline A40 | `$196,369.14`; costo de retorno `8.678%`; reducción de default `7.9025` pp | +| Auditoría pre-OOT | diciembre: default `0.145650`, miscoverage `0.124925`; estabilidad no implica cobertura seleccionada | Hashes SHA256 de los artefactos críticos están en [`EXTRACTION_MANIFEST.json`](EXTRACTION_MANIFEST.json). Verifica con `just validate-champion` o el skill `/crpto-validate-champion`. +El rebaseline y la frontera pool93 anterior se conservan como procedencia +congelada, no como claims activos del manuscrito IJDS. ## Requisitos del sistema @@ -60,6 +65,7 @@ just book-clean # borra _book/, _freeze/, .quarto/ # Pipeline de paper (no toca el champion) just paper-export # tablas + figuras + evidence + journal + libro +just ijds-evidence # A35--A40 y gobernanza de la política IJDS activa just tables # solo CSVs just figures # solo PNGs/PDFs @@ -67,8 +73,13 @@ just figures # solo PNGs/PDFs just lint # ruff check + format check just fmt # ruff fix + format just type-check # mypy src scripts +just type-advisory # ty sobre ruta activa IJDS, no bloqueante +just type-advisory-full # ty sobre src/scripts completos; bloquea el cierre IJDS +just api-docs-core # pdoc local para modulos core, salida ignorada +just hooks-check # valida hooks con pre-commit y prek just smoke # tests críticos rápidos just test # suite completa +just submission-check # cierre IJDS: claims, lint, type, suite completa, champion y PDF oficial # DVC just dvc-status # drift detection @@ -146,7 +157,7 @@ just paper-submission │ └── apa.csl # estilo APA 7 ├── crpto/ # paquete público mínimo (`import crpto`) ├── src/ # módulos fuente históricos (data, features, models, optimization, evaluation, utils) -├── scripts/ # 40+ entry points +├── scripts/ # entry points; ver scripts/README.md para rutas IJDS vs históricas ├── tests/ # 26 archivos pytest (markers slow / integration) ├── configs/ # YAML (pd_model, conformal, optimization, fairness, mrm) ├── dbt_project/ # 3 staging + 3 marts sobre crpto.duckdb diff --git a/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf b/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf index b547184..106f664 100644 Binary files a/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf and b/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf differ diff --git a/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.pdf b/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.pdf index 97d6814..370e4a9 100644 Binary files a/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.pdf and b/book/assets/figures/publication/crpto_fig13_alpha_gamma_funded_set.pdf differ diff --git a/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.pdf b/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.pdf index 816f9df..28f5f57 100644 Binary files a/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.pdf and b/book/assets/figures/publication/crpto_fig14_robust_region_heatmap.pdf differ diff --git a/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf b/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf index 89f5a6d..7e7d9de 100644 Binary files a/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf and b/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf differ diff --git a/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf b/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf index 447bd40..9162529 100644 Binary files a/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf and b/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf differ diff --git a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.pdf b/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.pdf index 790c211..2c0bf99 100644 Binary files a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.pdf and b/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.pdf differ diff --git a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png b/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png index f36d241..966ab67 100644 Binary files a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png and b/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png differ diff --git a/book/chapters/00-crpto-en-una-pagina.qmd b/book/chapters/00-crpto-en-una-pagina.qmd index 3f06a20..5adaa3e 100644 --- a/book/chapters/00-crpto-en-una-pagina.qmd +++ b/book/chapters/00-crpto-en-una-pagina.qmd @@ -19,7 +19,7 @@ se vuelve una restricción explícita de la decisión. | PD calibrada | Produce probabilidades auditables para el OOT de Lending Club. | AUC `0.7139`, Brier `0.1544`, ECE `0.0070`. | | Conformal Mondrian | Convierte la PD puntual en intervalos sobre la escala de riesgo. | Coverage 90% `0.9297`, min-group `0.9190`. | | LP robusto | Usa la señal conformal superior para financiar préstamos bajo presupuesto y tolerancia de riesgo. | Budget `$1M`, `tau = 0.1715`, `gamma = 0.5475`, capped blended uncertainty. | -| Certificado exacto | Audita el funded set promovido sobre 276,869 préstamos OOT. | Return `$184,832.48`, `V = 0.035350`, `Gamma_CP = 0.162616`, Markov cap `0.345084`, alpha-grid pass `8/8`. | +| Certificado exacto | Audita el funded set promovido sobre 276,869 préstamos OOT. | Return `$184,832.48`, `V = 0.035350`, `Gamma_CP = 0.162616`, `Gamma_res = 0.073584`, umbral Markov `0.345084`, pass `8/8`. | | Réplica externa | Aplica la receta congelada a Prosper y Freddie/Mendeley sin reabrir Lending Club. | Prosper coverage90 `0.9205`, Freddie coverage90 `0.9745`; robust LP positivo en ambos. | : Respuesta CRPTO en cuatro capas auditables. @@ -41,8 +41,9 @@ defendible es más específico: | `tau` | Techo de riesgo ponderado que el comité acepta para el portafolio. | | `gamma` | Parámetro de policy que decide cuánto castiga la incertidumbre en el LP. | | `Gamma_CP` | Prima conformal realizada: cuánto riesgo adicional reconoce el funded set frente a la PD puntual. | +| `Gamma_res` | Parte de la prima que el score efectivo no internaliza y que recupera el endpoint exacto. | | `V(alpha)` | No-cobertura ponderada observada del funded set. | -| `price_of_robustness` | Retorno esperado (point-PD) de la baseline no-robusta menos el de la policy robusta; con signo. En el champion vale `-$14,465.69` (negativo: la policy robusta supera a la baseline). | +| `price_of_robustness` | A40 compara dos decisiones emparejadas: CRPTO cede `5.875%` de retorno realizado y reduce `8.305` pp de default/V y `43.55` pp del umbral Markov. | : Objetos que convierten el resultado CRPTO en una conversación de comité. diff --git a/book/chapters/04-resultados.qmd b/book/chapters/04-resultados.qmd index 6c3129f..c024487 100644 --- a/book/chapters/04-resultados.qmd +++ b/book/chapters/04-resultados.qmd @@ -2,7 +2,7 @@ Los resultados del CRPTO deben leerse como una prueba de viabilidad del marco, no como un torneo aislado de AUC. La evidencia relevante está en la interacción entre incertidumbre y decisión. -Nota dual-tag (post-promoción pool93, 2026-07-02): este capítulo documenta la **cadena upstream congelada** (rebaseline `ijds-rebaseline-2026-06-07`, retorno `$170,464.54`), que hoy actúa como floor declarado. El body claim del paper IJDS es el punto pool93 (`$184,832.48`, `V=0.035350`, `Γ_CP=0.162616`, Markov cap `0.345084`, `8/8` alpha grid) sobre los mismos intervalos conformal congelados; ver tablas A35--A39 y `tests/test_pool93_body_claim_sync.py`. +Nota de gobernanza (actualizada 2026-07-09): este capítulo conserva la **cadena upstream congelada** (rebaseline `ijds-rebaseline-2026-06-07`, retorno `$170,464.54`) como floor declarado. El body claim IJDS es el punto pool93 (`$184,832.48`, `V=0.035350`, `Γ_CP=0.162616`, `Γ_res=0.073584`, umbral Markov `0.345084`, `8/8`) sobre los mismos intervalos; ver A35--A40 y `tests/test_pool93_body_claim_sync.py`. ### Base predictiva e incertidumbre El pipeline base sobre el champion monotónico confirmatorio sigue aportando la base regulatoria y probabilística del sistema: @@ -341,19 +341,21 @@ El comparador theorem-tight sigue siendo metodológicamente valioso porque muest - `gamma_cp = 0.160299` - y retorno todavía fuerte (`$166.3K`) -#### Instanciación empírica del Corolario 1 (Price of Robustness) - -El `cor-por` establece que $\text{PoR}(\alpha) \lesssim \Gamma_{\text{CP}}(\alpha) \cdot \overline{LGD} / \bar{r}$. Con los números del champion economic instanciamos: - -$$ -\text{PoR}_{\text{teórico}}(\alpha = 0.01) \;\lesssim\; \frac{\Gamma_{\text{CP}} \cdot \overline{LGD}}{\bar{r}} \;=\; \frac{0.187987 \cdot 0.45}{\bar{r}_{\text{funded}}} -$$ - -donde $\bar{r}_{\text{funded}}$ es el retorno neto promedio del funded set, y el corollary acota el precio **conceptual** de robustez (el techo de cuánto *podría* costar la prudencia conformal). - -El campo congelado `price_of_robustness` mide otra cosa, y conviene leer el signo con cuidado: se define como el retorno esperado (point-PD) de la baseline no-robusta menos el de la policy robusta. Para el champion vale `-$14,465.69` (`price_of_robustness_pct = -10.56%`, con la baseline no-robusta esperada de `$137,014.58` como denominador). El signo **negativo** es informativo: en esta evaluación la policy robusta no sacrifica retorno esperado frente a la baseline no-robusta —lo supera en `$14,465.69`— y el retorno realizado OOT (`$170,464.54`) queda todavía por encima de su propia esperanza point-PD (`$151,480.27`). Es decir, el funded set conformal-robusto es económicamente competitivo —de hecho superior— mientras carga además el certificado exacto a $\alpha = 0.01$. La cota teórica del corollary sigue siendo **conservadora pero no vacua** como techo conceptual; el precio empírico firmado simplemente resultó favorable aquí. - -El comparador theorem-tight (gamma=0.55) baja $\Gamma_{\text{CP}}$ a `0.160299` con un retorno menor (`$166,270`) y un `price_of_robustness_pct` de `-7.59%` (menos favorable, más cerca de cero que el `-10.56%` del economic champion). La frontera del corollary se instancia limpiamente: menor presupuesto conformal $\Gamma_{\text{CP}}$ → certificado más ajustado → menor retorno, con la ventaja económica firmada acercándose a cero. Las tres policies (economic, balanced, theorem-tight) ordenan esa frontera empírica dentro de la región robusta. +#### Baseline point-PD emparejada (A40) + +El campo congelado `price_of_robustness=-10.56%` no debe usarse: su baseline +histórica heredaba `pd_high`. A40 resuelve un control point-PD con los mismos +276,869 candidatos, presupuesto, concentración, `tau=0.1715`, LGD y solver que +la decisión seleccionada. El control realiza `$196,369.14` y CRPTO +`$184,832.48`: costo `$11,536.66` (`5.875%`). A cambio, default/V ponderado baja +de `0.118400` a `0.035350` y el umbral Markov exacto de `0.780579` a `0.345084`. +Esta es la comparación económica activa; no es evidencia causal ni universal. + +El comparador theorem-tight (`gamma=0.55`) conserva su lectura válida en los +ejes que sí fueron calculados sobre la policy robusta: baja +$\Gamma_{\text{CP}}$ a `0.160299` con un retorno realizado menor (`$166,270`). +La frontera defendible es retorno--bound--$\Gamma_{\text{CP}}$; los precios +históricos relativos a la baseline retirada no se usan para ordenarla. ::: {.column-page} ![Comparación de decisión regret entre three-stage, SPO+ end-to-end y CRPTO (Conformal Robust). En el artefacto local de la figura, SPO+ logra 49.09% menos regret que two-stage por diseño; CRPTO paga un precio de auditabilidad mayor en regret a cambio de garantías formales de cobertura.](../assets/figures/publication/crpto_fig9_spo_regret.png){#fig-p1-spo-regret fig-alt="Comparación de regret: two-stage vs SPO+ vs CRPTO para el CRPTO."} diff --git a/book/chapters/05-discusion.qmd b/book/chapters/05-discusion.qmd index ca41042..5fe0588 100644 --- a/book/chapters/05-discusion.qmd +++ b/book/chapters/05-discusion.qmd @@ -152,7 +152,7 @@ El body claim del paper IJDS es el punto pool93 (`$184,832.48`, `V=0.035350`, |---|---|---| | Champion upstream congelado = economic champion (histórico, floor) | `models/final_project_promotion.json`, `models/champion_portfolio_policy.json` | `tests/test_crpto_final_sync.py` | | Retorno de la cadena congelada `$170.5K` (= floor pool93) | `final_champion.realized_total_return` | Tabla `crpto_table0_key_metrics.csv` y DVC metric `crpto.final.robust_return` | -| Body point IJDS pool93 `$184,832.48`, `8/8` alpha grid | `pool93_ijds_consolidated_governance.json` (`selected_candidates.paper_body`), tablas A35--A39 | `tests/test_pool93_body_claim_sync.py` | +| Body point IJDS pool93 `$184,832.48`, `8/8` y baseline A40 | gobernanza `certificate-semantics-v2`, tablas A35--A40 | `tests/test_pool93_body_claim_sync.py` | | `alpha01_exact_pass=true` | `final_champion.alpha01_exact_pass` | Guardrail de sync del CRPTO | | `V=0.028875`, `gamma_cp=0.187987`, `violation=0` | `final_champion` y bound eval `276k` | Guardrail de sync + tablas paper-facing | | Región robusta `45/45` | `robust_region_summary` | DVC metrics `crpto.final.robust_region_*` | diff --git a/book/chapters/06-blueprint-manuscrito.qmd b/book/chapters/06-blueprint-manuscrito.qmd index f464217..70b1ce4 100644 --- a/book/chapters/06-blueprint-manuscrito.qmd +++ b/book/chapters/06-blueprint-manuscrito.qmd @@ -44,8 +44,8 @@ Un abstract posible, todavía largo para paper, sería: > Mondrian conformal prediction intervals into robust portfolio constraints. > On a 276,869-loan out-of-time Lending Club evaluation, the selected pool93 body > point earns `$184.8K` on a `$1M` budget while passing the declared eight-level -> alpha grid (`V=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084`, zero -> violation). The consolidated finite policy-grid frontier contains 50,010 +> alpha grid (`V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, exact +> Markov threshold `0.345084`, zero realized risk-tolerance excess). The consolidated finite policy-grid frontier contains 50,010 > deduplicated semantic policies, with 27,508 all-alpha above-floor policies, > showing that the result is not a single-point artifact. The contribution is > not higher AUC, but a reproducible bridge from calibrated probabilistic @@ -64,7 +64,7 @@ robusta verificable, no en otro leaderboard predictivo. | C2 | La capa conformal Mondrian produce cobertura útil y trazable para decisión. | Empírico-conformal | Coverage 90 `92.97%`, min group coverage `91.90%`, Winkler `1.111` | | C3 | El intervalo conformal se puede mapear a un conjunto de incertidumbre usado por un LP robusto. | Metodológico | Definiciones `u_i(alpha)`, `Gamma_CP`, policy modes | | C4 | El bound controla no-cobertura ponderada del funded set bajo supuestos distribution-free. | Teórico | `thm-conformal-feasibility`, Markov, `V <= sqrt(alpha)` | -| C5 | La policy oficial es el punto pool93 body/default, no el endpoint de máximo retorno. | Empírico-editorial | A35/A36, retorno `$184.8K`, Markov cap `0.345084`, `alpha_grid_pass=8/8` | +| C5 | La policy oficial es el punto pool93 body/default, no el endpoint de máximo retorno. | Empírico-editorial | A35/A40, retorno `$184.8K`, umbral Markov `0.345084`, `alpha_grid_pass=8/8` | | C6 | La frontera finita A35 muestra que el resultado no es un punto aislado. | Empírico | 50,010 políticas semánticas; 27,508 all-alpha above-floor policies | | C7 | La evidencia CRPTO y journal-package muestra robustez adicional sin cambiar la dirección del paper. | Robustez | A3--A36, Figuras 12--25 | @@ -130,7 +130,7 @@ selección sugerida es: | Figura 1: pipeline CRPTO | Cuerpo | Explica el aporte en una sola vista IJDS | `crpto_fig1_journal_pipeline.png` | | Figura bound: pila de claim | Teoría | Separa endpoint conformal, identidad determinística, supuesto ponderado y certificado exacto | `crpto_fig20_bound_claim_layers.png` | | Figura 2: alpha -> `Gamma_CP` -> funded set | teoría/método | Une parámetro conformal y decisión | `crpto_fig13_alpha_gamma_funded_set.png` | -| Tabla A35/A36 | Resultados | Muestra frontera finita y composición del funded set promovido | `crpto_tableA35_pool93_ijds_frontier.csv`, `crpto_tableA36_pool93_body_funded_grade_audit.csv` | +| Tablas A35/A40 | Resultados | Muestran la frontera finita y el baseline point-PD emparejado | `crpto_tableA35_pool93_ijds_frontier.csv`, `crpto_tableA40_pool93_point_baseline.csv` | | Tabla 1: métricas core | Resultados | Fija PD, CP y portfolio sin mezclar familias | `crpto_table0_key_metrics.csv` | | Tabla 2: champion y comparadores | Resultados | Economic vs theorem-tight vs balanced | `crpto_table1_champion_policy.csv` y A13 | | Tabla 3: robustez P1 | Apéndice corto | Post-selección, temporal, selector, shift | A3--A11 | @@ -141,7 +141,7 @@ Selección de figuras y tablas para el manuscrito. El paper no necesita mostrar todas las tablas en el cuerpo. El cuerpo debe proteger tres mensajes: la metodología, el punto pool93 promovido y la frontera -finita A35/A36. +finita A35 y la baseline A40. El appendix puede cargar la evidencia de stress y trazabilidad. ### Paquetes del online supplement (A--F) @@ -254,8 +254,8 @@ Una versión fuerte y honesta de las contribuciones sería: set y separamos explícitamente el tightening condicional de la garantía principal. 3. Mostramos que el enfoque produce una policy pool93 promovida con retorno - `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084`, cero - violación y pass `8/8`. + `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, umbral + Markov `0.345084`, exceso realizado cero y pass `8/8`. 4. Documentamos una frontera finita consolidada con 50,010 políticas semánticas y 27,508 all-alpha above-floor policies, por lo que el resultado no depende de un único punto elegido después de ver los datos. diff --git a/book/chapters/06b-guia-editorial-claims.qmd b/book/chapters/06b-guia-editorial-claims.qmd index 53b7ed7..5bbca67 100644 --- a/book/chapters/06b-guia-editorial-claims.qmd +++ b/book/chapters/06b-guia-editorial-claims.qmd @@ -46,7 +46,7 @@ empíricos, otros son de ingeniería reproducible. | 3 | El intervalo puede convertirse en conjunto de incertidumbre | Definición de `u_i(alpha)` y `Gamma_CP` | No tratar `Gamma_CP` como presupuesto ad hoc | | 4 | El bound controla no-cobertura ponderada del funded set | @sec-thm-feasibility y `V` | No decir que controla directamente una PD latente sin supuesto adicional | | 5 | Existe una frontera finita robusta en OOT | A35: 50,010 políticas semánticas; 27,508 all-alpha above-floor policies | No decir que eso elimina toda incertidumbre post-selección ni que certifica una región continua | -| 6 | El punto oficial es pool93 body/default, no el endpoint económico extremo | Retorno `$184.8K`, `V=0.035350`, `gamma_cp=0.162616`, Markov cap `0.345084`, pass `8/8` | No mezclarlo con el max-return endpoint de `$223K` ni con el punto de cap mínimo | +| 6 | El punto oficial es pool93 body/default, no el endpoint económico extremo | Retorno `$184.8K`, `V=0.035350`, `gamma_cp=0.162616`, `gamma_res=0.073584`, umbral Markov `0.345084`, pass `8/8` | No mezclarlo con el max-return endpoint de `$223K` ni con el punto de threshold mínimo | | 7 | La evidencia CRPTO fortalece el paper sin cambiar dirección | A7--A11, apéndice condicional, A25--A34 | No venderlo como validación live o universal | : Escalera editorial de claims del CRPTO {#tbl-crpto-claim-ladder} @@ -128,7 +128,7 @@ checkpoint MRM/fair-lending sin convertirlo en un claim legal. | `V` | No-cobertura ponderada del funded set | Cuánto default realizado quedó fuera de la cota | | `violation` | Exceso de riesgo ponderado sobre `tau` | Incumplimiento del límite de portafolio | | `alpha01_exact_pass` | Check exacto de bound en OOT | Semáforo para promoción paper-facing | -| `price_of_robustness` | Retorno esperado baseline no-robusta menos robusta (con signo; `-$14,465.69` en el champion) | Costo/beneficio económico de la robustez, auditable | +| `price_of_robustness` | A40 empareja candidatos y restricciones: costo realizado `5.875%`, reducción default/V `8.305` pp y reducción de threshold `43.55` pp | Costo económico de la robustez, solo con comparadores semánticamente equivalentes | : Diccionario operativo del CRPTO {#tbl-crpto-operational-dictionary} @@ -202,7 +202,7 @@ improvisar: si llega una objeción nueva, primero se busca aquí. |---|---|---| | Diagrama CRPTO | Figura 1 | Explica el método en una sola vista | | Escalera alpha -> `Gamma_CP` -> funded set | Teoría / método | Une conformal y robust optimization | -| Tabla A35/A36 | Resultados principales | Demuestra frontera finita y audit de composición del funded set | +| Tablas A35/A40 | Resultados principales | Demuestran frontera finita y trade-off contra point-PD emparejado | | Holdout temporal estricto CRPTO | Appendix o robustness checks | Responde post-selección | | Funded-set composition | Appendix | Útil para reviewers de crédito y riesgo | | A10 exact finalists | Appendix | Defiende el selector conformal | diff --git a/book/chapters/13-trazabilidad.qmd b/book/chapters/13-trazabilidad.qmd index 6f3762d..31b4fcb 100644 --- a/book/chapters/13-trazabilidad.qmd +++ b/book/chapters/13-trazabilidad.qmd @@ -7,7 +7,7 @@ page-layout: article Esta página es el **índice maestro** del CRPTO. Cumple un rol específico que ningún chapter narrativo cumple: dar a un revisor (académico, regulatorio o interno) la capacidad de tomar cualquier claim del paper, encontrar el artefacto exacto que lo soporta, identificar el script generador y verificar el guardrail que mantiene la coherencia. Es el contrato de trazabilidad operativa del proyecto. -Nota dual-tag (post-promoción pool93, 2026-07-02): los claims C1--C7 de esta página trazan la **cadena upstream congelada** (rebaseline `ijds-rebaseline-2026-06-07`, retorno `$170,464.54`, hoy floor declarado). El body claim del paper IJDS es el punto pool93 (`$184,832.48`, `8/8` alpha grid), anclado en `pool93_ijds_consolidated_governance.json` y las tablas A35--A39, con guardrail `tests/test_pool93_body_claim_sync.py`. +Nota de gobernanza (actualizada 2026-07-09): los claims C1--C7 trazan la **cadena upstream congelada** (rebaseline `ijds-rebaseline-2026-06-07`, retorno `$170,464.54`, floor declarado). El body claim IJDS es el punto pool93 (`$184,832.48`, `8/8`), anclado en la gobernanza `certificate-semantics-v2` y las tablas A35--A40, con guardrail `tests/test_pool93_body_claim_sync.py`. | Claim del paper | Artefacto que lo ancla | Guardrail que lo protege | Lectura rápida | |---|---|---|---| diff --git a/book/chapters/14-release.qmd b/book/chapters/14-release.qmd index 132c3c8..0cbfda8 100644 --- a/book/chapters/14-release.qmd +++ b/book/chapters/14-release.qmd @@ -128,7 +128,10 @@ Manifest de uso por sección del capítulo 14. | A31 | External OOT subperiod metrics | No | Sí | Sí | Prosper por año y Freddie por quarter; registra el borde 2015Q4. | | A32 | Prosper default-definition sensitivity | No | Sí | Sí | Verifica que la definición de default no decide sola el claim Prosper. | | A33 | Freddie segment sensitivity | No | Sí | Sí | Combined/green pasan alpha01; red queda como sensibilidad honesta. | -| A34 | Cross-dataset price of robustness | Sí o appendix corto | Sí | Sí | Muestra que el premium externo crece con default rate y que LC seleccionado queda como referencia favorable. | +| A34 | Cross-dataset price of robustness | Sí o appendix corto | Sí | Sí | Muestra que el premium externo crece con default rate; no reutiliza el comparador histórico retirado de Lending Club. | +| A35 | Policy-aware finite-grid frontier | Sí | Sí | Sí | Usa endpoint exacto para políticas lineales, capped y tail; no es óptimo continuo. | +| A36--A39 | Selected-allocation audits | No | Sí | Sí | Composición, cola, concentración y bootstrap de la asignación promovida. | +| A40 | Matched point-PD baseline | Sí | Sí | Sí | Mismos candidatos y restricciones operativas; cuantifica costo de retorno y reducción de riesgo. | Manifest de tablas para decidir cuerpo, appendix y tesis. ::: @@ -158,7 +161,7 @@ Manifest de tablas para decidir cuerpo, appendix y tesis. | Fig 22 | `crpto_fig22_external_replication` | Sí | Sí | Sí | Resume gates conformales y valor LP robusto en Prosper/Freddie. | | Fig 23 | `crpto_fig23_external_candidate_sensitivity` | No | Sí | Sí | Verifica que la réplica externa no depende de un pool pequeño. | | Fig 24 | `crpto_fig24_freddie_all_candidate_certificate` | Sí o appendix corto | Sí | Sí | Certificado all-candidate Freddie sobre `1,396,053` candidatos OOT. | -| Fig 25 | `crpto_fig25_price_of_robustness_scaling` | Sí o appendix corto | Sí | Sí | Conecta default rate, premium de robustez y contraste con el champion LC seleccionado. | +| Fig 25 | `crpto_fig25_price_of_robustness_scaling` | Sí o appendix corto | Sí | Sí | Conecta default rate y premium de robustez exclusivamente en las aplicaciones externas congeladas. | | Fig alpha-gamma | `crpto_fig_alpha_gamma_bound` | Quizá | Sí | Sí | Figura técnica del bound. | Manifest de figuras para extraer paper y appendix. diff --git a/book/chapters/16-fundamentos-conformal-optimizacion.qmd b/book/chapters/16-fundamentos-conformal-optimizacion.qmd index e520edd..beb3d9c 100644 --- a/book/chapters/16-fundamentos-conformal-optimizacion.qmd +++ b/book/chapters/16-fundamentos-conformal-optimizacion.qmd @@ -647,8 +647,15 @@ En la configuración del proyecto (`configs/optimization.yaml`), el parámetro ` ::: ::: {.callout-note} -## El signo importa: en este proyecto el precio salió favorable -La guía anterior describe el caso *típico*, donde la robustez cuesta retorno. El campo congelado `price_of_robustness` se define con signo: retorno esperado de la baseline no-robusta menos el de la policy robusta. Para el champion vale `-$14,465.69` (`-10.56%`) — **negativo** — porque el funded set conformal-robusto promovido no solo no sacrifica retorno esperado frente a la baseline, sino que lo supera (ver la instanciación empírica del Corolario 1 en @sec-crpto-results). Un PoR negativo no contradice la teoría: el corollary acota el precio *conceptual* máximo; el precio *medido* sobre este funded set resultó a favor de la prudencia. +## El signo exige una baseline point-PD emparejada +El campo congelado `price_of_robustness=-10.56%` queda retirado porque la +baseline histórica etiquetada como no robusta heredaba la restricción +`pd_high`. A40 reconstruye una baseline point-PD con los mismos 276,869 +candidatos, presupuesto, concentración, `tau=0.1715`, LGD y solver. CRPTO cede +`5.875%` de retorno realizado, reduce default/V ponderado en `8.305` puntos +porcentuales y baja el umbral Markov exacto en `43.55` puntos. Esta comparación +no cambia el certificado seleccionado; aclara su costo económico contra un +control semánticamente equivalente. ::: La frontera de Pareto entre retorno y robustez se explora en el script `scripts/optimize_portfolio_tradeoff.py`, que genera el *tradeoff curve* variando el parámetro $\gamma$ y registrando el PoR resultante. diff --git a/book/chapters/30-replicacion-multidataset.qmd b/book/chapters/30-replicacion-multidataset.qmd index ef40c25..e9689ea 100644 --- a/book/chapters/30-replicacion-multidataset.qmd +++ b/book/chapters/30-replicacion-multidataset.qmd @@ -140,13 +140,14 @@ condicional sería excesiva. | A32 Prosper default sensitivity | Main, `Defaulted` only y `Chargedoff` only pasan 90% y alpha01, todos con LP robusto positivo. | La definición de default no decide por sí sola el claim Prosper. | | A33 Freddie red/green | `both` y `green` pasan alpha01; `red` falla alpha01 (`0.9850`) aunque mantiene coverage90 y LP positivo. | FM48 combinado es el claim promovido; red queda como sensibilidad honesta. | -## El precio de robustez crece con el riesgo del panel +## El precio de robustez en las aplicaciones externas -La réplica externa no solo confirma que los gates se sostienen: revela un patrón -económico que un champion de un solo dataset no podía mostrar. El campo -`price_of_robustness_pct` --definido con el mismo signo que en Lending Club, como +La réplica externa no solo confirma que los gates se sostienen: muestra un patrón +económico que un solo dataset no podía revelar. El campo +`price_of_robustness_pct` --definido como `(no-robusto − robusto) / no-robusto`-- es **positivo** en las aplicaciones -congeladas externas y **crece monótonamente con la tasa de default del panel**. +congeladas externas y queda ordenado por la tasa de default del panel en los +cuatro casos observados. | Aplicación congelada | Default del panel | AUC | Precio de robustez | |---|---:|---:|---:| @@ -159,7 +160,7 @@ congeladas externas y **crece monótonamente con la tasa de default del panel**. Todas usan la misma receta sin búsqueda de champion. Fuente: `crpto_tableA34_price_of_robustness_cross_dataset.csv`. -![El precio de robustez crece con la tasa de default del panel: las aplicaciones congeladas (Freddie green/combinado/red, Prosper) forman una serie positiva y monótona, mientras el champion seleccionado de Lending Club queda como referencia favorable por debajo de cero.](../assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png){#fig-book-price-scaling width="85%" fig-alt="Gráfico de líneas en eje x logarítmico: el precio de robustez sube de +1.00% a +9.46% al crecer la tasa de default del panel; Lending Club aparece como línea de referencia en -10.56%."} +![En las cuatro aplicaciones externas congeladas, el precio de robustez es positivo y queda ordenado por la tasa de default del panel.](../assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png){#fig-book-price-scaling width="85%" fig-alt="Gráfico de líneas en eje x logarítmico: el precio de robustez aumenta de +1.00% a +9.46% entre cuatro aplicaciones externas congeladas ordenadas por tasa de default."} La lectura tiene base teórica limpia: más riesgo de default ensancha los intervalos conformales, así que el peor caso robusto descuenta más retorno y la @@ -169,18 +170,18 @@ prima por comprar la garantía de cobertura sube. Dentro de Freddie, el segmento sí sola --`green` y `red` tienen AUC casi idéntico pero distinta prima--: lo que manda es el riesgo irreducible del panel. -Lending Club es la excepción informativa: su precio de robustez es **favorable** -(`−10.56%`), pero ese número proviene del champion *seleccionado* por la búsqueda -bound-aware, no de una aplicación ciega de la receta. La selección encontró -precisamente un funded set robusto que además gana en retorno esperado. Reportar -ambas cosas --la prima positiva acotada bajo aplicación congelada y el caso -favorable bajo selección-- es más honesto y más fuerte que afirmar que la robustez -es siempre gratis o siempre cara. +Lending Club no entra en esa serie. El campo histórico `−10.56%` comparaba la +policy robusta contra una baseline rotulada `nonrobust` que todavía heredaba una +restricción de endpoint conformal. La auditoría point-PD corregida en +`tau=0.1715` es A40: CRPTO cede `5.875%` de retorno realizado y reduce `8.305` +puntos de default/V frente al control. No se mezcla con las cuatro aplicaciones +externas porque usa otro contrato. El claim activo de Lending Club se apoya en +la frontera policy-aware A35 y la baseline emparejada A40. -El mensaje editorial es directo: **la robustez nunca es económicamente -catastrófica**. Bajo aplicación congelada la prima se queda en un dígito a -dígito-bajo-doble, y bajo selección puede volverse favorable. CRPTO mide en qué -régimen está cada panel en vez de asumir un costo fijo de robustez. +El mensaje editorial defendible es más acotado: **en las cuatro aplicaciones +externas congeladas, la prima observada va de un dígito a bajo-doble-dígito**. +Cuatro puntos ordenados son compatibles con el mecanismo, pero no demuestran una +ley general ni que la robustez sea siempre barata. ## Qué mejora en el paper @@ -192,9 +193,9 @@ la receta conserva gates conformales y valor LP positivo en Prosper y Freddie". Ese matiz mejora tres claims: 1. El método no parece depender de una idiosincrasia de Lending Club. -2. El precio de robustez deja de ser un número aislado de Lending Club y se - vuelve un patrón interpretable: una prima positiva acotada que crece con el - riesgo del panel, comparable entre un P2P de consumo y un panel hipotecario. +2. El precio de robustez se vuelve un patrón externo interpretable: una prima + positiva acotada y ordenada por riesgo en los cuatro paneles observados, + comparable entre un P2P de consumo y un panel hipotecario. 3. Home Credit se descarta por criterio económico explícito, no por conveniencia de resultados. diff --git a/book/includes/_build-info.qmd b/book/includes/_build-info.qmd index 692731c..c4dbb67 100644 --- a/book/includes/_build-info.qmd +++ b/book/includes/_build-info.qmd @@ -1,3 +1,3 @@ ::: {.build-info} -Build: `ede5dd1` | Rama: `main` | Actualizado: `2026-06-09` +Build: `2a9b5e9` | Rama: `codex/ijds-refactor-lab-2026-07-08` | Actualizado: `2026-07-09` ::: diff --git a/book/index.qmd b/book/index.qmd index c775cf6..1fe0819 100644 --- a/book/index.qmd +++ b/book/index.qmd @@ -20,14 +20,15 @@ ![Las seis etapas del pipeline CRPTO --- datos y split temporal, PD calibrada, conformal Mondrian, conjunto de incertidumbre, LP robusto y política promovida con bound exacto --- anotadas con métricas del cierre IJDS pool93: retorno $184,832.48, $V=0.035350$, $\Gamma_{\text{CP}}=0.162616$ y frontera finita A35/A36.](assets/figures/editorial/diagrama-crpto.png){#fig-crpto-overview fig-alt="Diagrama maestro del pipeline CRPTO en seis etapas desde datos hasta política promovida auditable."} La superficie IJDS activa promueve el punto pool93 body/default: retorno -`$184,832.48`, `V = 0.035350`, `Gamma_CP = 0.162616`, Markov cap `0.345084`, -`alpha_grid_pass = 8/8` y frontera finita A35/A36. Los cierres anteriores quedan +`$184,832.48`, `V = 0.035350`, `Gamma_CP = 0.162616`, `Gamma_res = 0.073584`, +umbral Markov `0.345084`, `alpha_grid_pass = 8/8` y frontera finita A35. A40 +cuantifica el trade-off contra point-PD. Los cierres anteriores quedan como procedencia histórica, no como baseline operativo vigente. ::: {.callout-warning} Algunos capítulos largos de tesis conservan análisis históricos del champion anterior para trazabilidad. La fuente paper-facing actual es -`paper/CRPTO_ijds.qmd`, su suplemento y las tablas A35/A36. +`paper/CRPTO_ijds.qmd`, su suplemento y las tablas A35--A40. ::: ## Mapa editorial @@ -40,8 +41,8 @@ superconjunto académico. | Superficie | Qué defiende | Evidencia visible | |---|---|---| -| Paper IJDS | El claim estrecho: una PD calibrada congelada puede convertirse en una decisión robusta y auditable mediante conformal prediction. | Teoría mínima, Figura 1, exact funded-set certificate, frontera finita A35/A36, frontera regret-auditability y réplica externa compacta. | -| Online supplement | Que el resultado no es frágil ni cherry-picked. | A3--A36, ablations, stress tests, MRM/fairness, funded-set audit, SPO+, tail risk, diagnósticos multi-distribución/online y Prosper/Freddie. | +| Paper IJDS | El claim estrecho: una PD calibrada congelada puede convertirse en una decisión robusta y auditable mediante conformal prediction. | Teoría mínima, Figura 1, frontera policy-aware A35, baseline A40, regret-auditability y réplica externa compacta. | +| Online supplement | Que el resultado no es frágil ni cherry-picked. | A3--A40, ablations, stress tests, MRM/fairness, funded-set audit, SPO+, tail risk, diagnósticos multi-distribución/online y Prosper/Freddie. | | Tesis | El superconjunto académico y operativo del paper. | Fundamentos, arquitectura reproducible, gobernanza, trazabilidad, réplica multidataset, bibliografía extendida y future work controlado. | | Reproducibilidad | Que las cifras vienen de artefactos versionados, no de edición manual. | DVC, manifests, tablas/figuras regenerables, tests de sincronía y guardrails de champion congelado. | diff --git a/book/references.bib b/book/references.bib index 3a9f6a8..ac527f5 100644 --- a/book/references.bib +++ b/book/references.bib @@ -1229,7 +1229,7 @@ @article{delage2010dro @inproceedings{zhao2016p2pportfolio, author = {Zhao, Hongke and Liu, Qi and Wang, Guifeng and Ge, Yong and Chen, Enhong}, - title = {Portfolio Selections in {P2P} Lending}, + title = {Portfolio Selections in {P2P} Lending: A Multi-Objective Perspective}, booktitle = {Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining}, pages = {2075--2084}, year = {2016}, @@ -1237,7 +1237,7 @@ @inproceedings{zhao2016p2pportfolio } @article{serrano2016profitscoring, - author = {Serrano-Cinca, Carlos and Gutierrez-Nieto, Begona}, + author = {Serrano-Cinca, Carlos and Guti{\'e}rrez-Nieto, Bego{\~n}a}, title = {The Use of Profit Scoring as an Alternative to Credit Scoring Systems in Peer-to-Peer ({P2P}) Lending}, journal = {Decision Support Systems}, volume = {89}, @@ -1254,3 +1254,167 @@ @inproceedings{liu2021riskbounds pages = {22083--22094}, year = {2021} } + +% --- T. Literature expansion scan 2026-07-08 --- + +@article{sadana2025contextual, + author = {Sadana, Utsav and Chenreddy, Abhilash and Delage, Erick and Forel, Alexandre and Frejinger, Emma and Vidal, Thibaut}, + title = {A Survey of Contextual Optimization Methods for Decision-Making Under Uncertainty}, + journal = {European Journal of Operational Research}, + volume = {320}, + number = {2}, + pages = {271--289}, + year = {2025}, + doi = {10.1016/j.ejor.2024.03.020} +} + +@article{xu2025profit_uncertainty_credit, + author = {Xu, Yong and Kou, Gang and Ergu, Daji}, + title = {Profit-Based Uncertainty Estimation with Application to Credit Scoring}, + journal = {European Journal of Operational Research}, + volume = {325}, + number = {2}, + pages = {303--316}, + year = {2025}, + doi = {10.1016/j.ejor.2025.03.007} +} + +@article{xu2024profit_risk_credit, + author = {Xu, Yong and Kou, Gang and Peng, Yi and Ding, Kexing and Ergu, Daji and Alotaibi, Fahd S.}, + title = {Profit- and Risk-Driven Credit Scoring Under Parameter Uncertainty: A Multiobjective Approach}, + journal = {Omega}, + volume = {125}, + pages = {103004}, + year = {2024}, + doi = {10.1016/j.omega.2023.103004} +} + +@inproceedings{farinhas2024nonexchangeable_crc, + author = {Farinhas, Antonio and Zerva, Chrysoula and Ulmer, Dennis and Martins, Andre F. T.}, + title = {Non-Exchangeable Conformal Risk Control}, + booktitle = {International Conference on Learning Representations}, + year = {2024}, + url = {https://arxiv.org/abs/2310.01262} +} + +@article{hegazy2025valid_selection_conformal_sets, + author = {Hegazy, Mahmoud and Aolaritei, Liviu and Jordan, Michael I. and Dieuleveut, Aymeric}, + title = {Valid Selection Among Conformal Sets}, + journal = {arXiv preprint arXiv:2506.20173}, + year = {2025}, + url = {https://arxiv.org/abs/2506.20173} +} + +@article{hullman2025conformal_human_decision, + author = {Hullman, Jessica and Wu, Yifan and Xie, Dawei and Guo, Ziyang and Gelman, Andrew}, + title = {Conformal Prediction and Human Decision Making}, + journal = {arXiv preprint arXiv:2503.11709}, + year = {2025}, + url = {https://arxiv.org/abs/2503.11709} +} + +@article{djeundje2025dynamic_loan_portfolio_profitability, + author = {Djeundje, Viani B. and Crook, Jonathan and Andreeva, Galina}, + title = {The Devil in the Details: Dynamic Prediction of Loan Portfolio Profitability with Macroeconomic Drivers Through Multi-State Modelling}, + journal = {European Journal of Operational Research}, + volume = {327}, + number = {2}, + pages = {703--715}, + year = {2025}, + doi = {10.1016/j.ejor.2025.07.008} +} + +@article{stratigakos2026decision_calibrated_sets, + author = {Stratigakos, Akylas and Wen, Honglin and Spyrou, Elina and Pinson, Pierre}, + title = {Decision-Calibrated Prediction Sets for Robust Power System Operations}, + journal = {arXiv preprint arXiv:2606.02081}, + year = {2026}, + url = {https://arxiv.org/abs/2606.02081} +} + +@article{chen2026polyhedral_conformal_ro, + author = {Chen, Shuyi and Zhou, Wenbin and Zhu, Shixiang}, + title = {Learning Polyhedral Conformal Sets for Robust Optimization}, + journal = {arXiv preprint arXiv:2605.08506}, + year = {2026}, + url = {https://arxiv.org/abs/2605.08506} +} + +@article{wang2026optimal_decision_prediction_sets, + author = {Wang, Tao and Dobriban, Edgar}, + title = {Optimal Decision-Making Based on Prediction Sets}, + journal = {arXiv preprint arXiv:2602.00989}, + year = {2026}, + url = {https://arxiv.org/abs/2602.00989} +} + +@article{huang2026oce_rcps, + author = {Huang, Jiayi and Farzaneh, Amirmohammad and Simeone, Osvaldo}, + title = {Optimized Certainty Equivalent Risk-Controlling Prediction Sets}, + journal = {arXiv preprint arXiv:2602.13660}, + year = {2026}, + url = {https://arxiv.org/abs/2602.13660} +} + +@article{baesens2026foundation_credit_risk, + author = {Baesens, Bart and Goethals, Andreas and Lessmann, Stefan and De Vos, Simon and Bravo, Cristian and Martens, David and Medina-Olivares, Victor and Mues, Christophe and Oskarsdottir, Maria and vanden Broucke, Seppe and Verdonck, Tim and Verbeke, Wouter}, + title = {Foundation Models for Credit Risk Prediction: A Game Changer?}, + journal = {arXiv preprint arXiv:2605.18147}, + year = {2026}, + url = {https://arxiv.org/abs/2605.18147} +} + +@article{distaso2025business_cycle_losses, + author = {Distaso, Walter and Roccazzella, Francesco and Vrins, Frederic}, + title = {Business Cycle and Realized Losses in the Consumer Credit Industry}, + journal = {European Journal of Operational Research}, + volume = {323}, + number = {3}, + pages = {1024--1039}, + year = {2025}, + doi = {10.1016/j.ejor.2024.12.026} +} + +@article{ballegeer2025explanation_stability, + author = {Ballegeer, Matteo and Bogaert, Matthias and Benoit, Dries F.}, + title = {Evaluating the Stability of Model Explanations in Instance-Dependent Cost-Sensitive Credit Scoring}, + journal = {European Journal of Operational Research}, + volume = {326}, + number = {3}, + pages = {630--640}, + year = {2025}, + doi = {10.1016/j.ejor.2025.05.039} +} + +@article{wiberg2025ai_or, + author = {Wiberg, Holly and Dai, Tinglong and Lam, Henry and Kulkarni, Radhika}, + title = {Synergizing Artificial Intelligence and Operations Research: Perspectives from {INFORMS} Fellows on the Next Frontier}, + journal = {INFORMS Journal on Data Science}, + volume = {5}, + number = {1}, + pages = {14--23}, + year = {2025}, + doi = {10.1287/ijds.2025.0077} +} + +@article{morucci2022robust_matching_uncertainty, + author = {Morucci, Marco and Noor-E-Alam, Md. and Rudin, Cynthia}, + title = {A Robust Approach to Quantifying Uncertainty in Matching Problems of Causal Inference}, + journal = {INFORMS Journal on Data Science}, + volume = {1}, + number = {2}, + pages = {156--171}, + year = {2022}, + doi = {10.1287/ijds.2022.0020} +} + +@article{chen2025cost_sensitive_adversarial, + author = {Chen, Qiyuan and Al Kontar, Raed and Nouiehed, Maher and Yang, X. Jessie and Lester, Corey}, + title = {Rethinking Cost-Sensitive Classification in Deep Learning via Adversarial Data Augmentation}, + journal = {INFORMS Journal on Data Science}, + volume = {4}, + number = {1}, + pages = {1--19}, + year = {2025}, + doi = {10.1287/ijds.2022.0033} +} diff --git a/configs/crpto_publication_targets.yaml b/configs/crpto_publication_targets.yaml index 3c0cd4a..0a9632d 100644 --- a/configs/crpto_publication_targets.yaml +++ b/configs/crpto_publication_targets.yaml @@ -1,4 +1,4 @@ -version: "2026-07-04" +version: "2026-07-09" project: paper-crpto decision_status: active @@ -36,11 +36,13 @@ primary_target: - "Freeze a release tag and reproducibility bundle after anonymity decision." current_paper_scope: include: - - "Pool93 finite-grid return-bound frontier as the active IJDS body claim." - - "Frozen upstream PD/calibration/conformal chain retained as provenance and return floor." - - "A3--A39 as supplement/appendix evidence generated from frozen artifacts." + - "Exact 90% conformal replay plus a November-selected 3x3 linear policy grid under deterministic endpoint cap B_u<=0.28." + - "Outcome-free December selector replay and independent decision audit showing that policy stability is not selected-set coverage validity." + - "One interpretable policy, q=(p+u)/2 with tau=0.17, and a matched point-PD comparator." + - "Frozen upstream PD/calibration/conformal chain and historical pool93 frontier retained only as provenance." + - "A3--A34 as supporting diagnostics and A35--A40 as the active exact-alpha/selector/evaluation bundle." - "Regret-auditability frontier as the body-level SPO+/CRPTO trade-off." - - "OCE/CVaR and robust satisficing as diagnostics, not new selectors." + - "OCE/CVaR, robust satisficing, external replications, and SPO+ as diagnostics or comparators, not additional CRPTO methods." - "Cluster-aware dependence caveat/proposition in the theory supplement." - "Prosper/Freddie external economic replication without reopening the Lending Club champion." - "Quarto/DVC/DagsHub/MLflow companion after anonymity handling." @@ -62,27 +64,34 @@ journal_strengthening_pack: requires_new_run: false tail_risk_oce_cvar_diagnostic: status: include_supplement - role: "Tail-risk audit of legacy diagnostic surfaces plus A37 selected pool93 allocation repricing under LGD alternatives." + role: "Legacy OCE/CVaR sensitivity retained in the supplement; it does not select or redefine the active linear policy." artifacts: - reports/crpto/tables/crpto_tableA12_tail_risk_oce_cvar.csv - reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv - - reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv requires_new_run: false - pool93_frontier_and_selected_allocation: + exact_alpha_calibration_selected_policy: status: include_body_and_supplement - role: "Active IJDS return-bound certificate: A35 frontier in the body, A36--A39 selected-allocation audits in the supplement." + role: "Active IJDS evidence: exact alpha replay, split nine-policy selector/audit, temporal/funded-set audits, month-cluster bootstrap, and matched comparisons." artifacts: - - reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv - - reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv - - reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv - - reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv - - reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json + - reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv + - reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv + - reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv + - reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv + - reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv + - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json + requires_new_run: false + matched_point_pd_baseline: + status: include_body_and_supplement + role: "A40 matched Lending Club baseline: same candidates, tau=0.17 and operating constraints; point PD replaces the conformal guardrail." + artifacts: + - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv + - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json requires_new_run: false robust_satisficing_margins: status: include_supplement_or_short_body - role: "OR/committee margins for return, V, Gamma_CP, zero violation and robust-region pass." + role: "OR/committee margins for return, V, Gamma_CP, realized risk-tolerance excess and robust-region pass." artifacts: - reports/crpto/tables/crpto_tableA13_satisficing_margins.csv requires_new_run: false @@ -96,7 +105,7 @@ journal_strengthening_pack: requires_new_run: false tail_satisficing_challenger_audit: status: include_supplement - role: "Journal-only legacy audit that ranks tail-satisficing alternatives without promoting them over the pool93 body point." + role: "Journal-only legacy audit; it is not a selector for the active 50/50 linear policy." artifacts: - reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv - models/crpto_tail_satisficing_audit_status.json @@ -228,4 +237,4 @@ current_decision: - operations_research - informs_ijoo reason: "CRPTO currently has the strongest fit as reproducible decision-focused data science with OR substance." - p2_p3_boundary: "P2/P3 are no longer a blanket exclusion: the journal strengthening pack enters as diagnostics, framing and theory caveat; method-changing extensions remain future work and are not acceptance criteria." + p2_p3_boundary: "P2/P3 are no longer a blanket exclusion: the journal strengthening pack enters as diagnostics, framing and theory caveat; method-changing extensions remain outside the submitted claim and are not acceptance criteria." diff --git a/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml b/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml new file mode 100644 index 0000000..d1fe691 --- /dev/null +++ b/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml @@ -0,0 +1,52 @@ +schema_version: "2026-07-09.7" +run_tag: "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" + +source: + upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" + conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" + candidate_path: "data/processed/test_fe.parquet" + exact_alpha_grid_path: "data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid.parquet" + +design: + alpha: 0.10 + budget: 1000000.0 + max_concentration: 0.25 + lgd: 0.45 + period_order: ["2018H1", "2018H2", "2019H1", "2019H2", "2020+"] + combine_years_from: 2020 + selection_period: "2017-11" + audit_period: "2017-12" + endpoint_budget_cap: 0.28 + selection_min_budget_utilization: 0.999 + selection_rule: >- + maximize expected point-PD objective on November 2017 under a deterministic + endpoint-budget cap of 0.28, the effective-PD cap, and full budget use; + reserve December 2017 for an outcome-free selector stability replay and a + post-selection decision audit + +policy_grid: + family: "simple_linear_conformal_guardrail" + risk_tolerances: [0.15, 0.17, 0.19] + gammas: [0.25, 0.50, 0.75] + uncertainty_aversions: [0.0] + +incumbent_policy: + risk_tolerance: 0.17 + gamma: 0.75 + uncertainty_aversion: 0.0 + +execution: + solver_backend: "highspy" + time_limit: 300 + threads: 1 + random_seed: 42 + +claim_boundary: >- + The final tagged rule selects among nine round-number policies on November + 2017 without exposing the selector to outcomes or assumption-conditional + statistics. December 2017 independently replays the outcome-free selector + and audits the already-fixed decision. The audit is diagnostic rather than a + selected-set coverage theorem. Earlier development inspected the static OOT + corpus, so the January 2018--September 2020 evaluation remains a transparent + retrospective lockbox replay, not a pristine prospective trial, causal + estimate, or live-deployment guarantee. diff --git a/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml b/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml new file mode 100644 index 0000000..6e98e4a --- /dev/null +++ b/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml @@ -0,0 +1,45 @@ +schema_version: "2026-07-09.6" +run_tag: "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6" + +source: + upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" + conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" + conformal_intervals_path: "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet" + exact_alpha_grid_path: "data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid.parquet" + +design: + alpha: 0.10 + budget: 1000000.0 + max_concentration: 0.25 + lgd: 0.45 + period_order: ["2018H1", "2018H2", "2019H1", "2019H2", "2020+"] + combine_years_from: 2020 + markov_threshold_cap: 0.60 + selection_min_budget_utilization: 0.999 + selection_rule: "maximize expected point-PD objective on the calibration holdout under a 0.60 endpoint-plus-Markov screen, the effective-PD cap, and full budget use" + +policy_grid: + family: "simple_linear_conformal_guardrail" + risk_tolerances: [0.15, 0.17, 0.19] + gammas: [0.25, 0.50, 0.75] + uncertainty_aversions: [0.0] + +incumbent_policy: + risk_tolerance: 0.17 + gamma: 0.75 + uncertainty_aversion: 0.0 + +execution: + solver_backend: "highspy" + time_limit: 300 + threads: 1 + random_seed: 42 + +claim_boundary: >- + The final tagged policy rule ranks nine round-number candidates on the + calibration development block without reading default, realized-return, or + other outcome-derived selector columns, and then freezes the selected policy + before OOT evaluation. Conformal endpoints themselves use calibration labels, + as required. Earlier project development inspected this static OOT corpus, so + the evaluation is a transparent retrospective lockbox replay rather than a + pristine prospective trial, causal estimate, or live-deployment guarantee. diff --git a/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml b/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml new file mode 100644 index 0000000..8f447b4 --- /dev/null +++ b/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml @@ -0,0 +1,19 @@ +schema_version: "2026-07-09.1" +run_tag: "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1" + +source: + upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" + conformal_namespace: "champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1" + conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" + conformal_intervals_path: "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet" + +design: + alpha_grid: [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] + alpha_mapping: "proportional frozen conservative ratio alpha_used/alpha_target from the selected 90% recipe" + replay_tolerance: 1.0e-12 + +claim_boundary: >- + Recomputes finite-sample Mondrian quantiles for every declared alpha while + freezing the selected partition, calibration fit/holdout split, score scale, + and holdout-learned widening factors. It is a retrospective OOT audit, not a + post-selection or live-deployment guarantee. diff --git a/docs/SCOPE_AND_GOVERNANCE.md b/docs/SCOPE_AND_GOVERNANCE.md index 199e900..5f0d253 100644 --- a/docs/SCOPE_AND_GOVERNANCE.md +++ b/docs/SCOPE_AND_GOVERNANCE.md @@ -25,26 +25,36 @@ CRPTO does not cover: - New model-training research unless it is explicitly isolated from the frozen champion or run under a new tag. -## Frozen champion contract - -The current IJDS paper-facing CRPTO body point is the promoted pool93 -finite-grid frontier closure: - -- run tag: `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` -- body/default policy mode: `capped_blended_uncertainty` (family - `claim_micro_ext_body_cap345`, selected from the consolidated frontier) -- robust return: `$184,832.48` -- `V(alpha=0.01)=0.035350` -- `Gamma_CP(alpha=0.01)=0.162616` -- Markov cap at `alpha=0.01`: `0.345084` -- exact alpha violation: `0.0` -- declared alpha-grid pass: `8/8` -- main paper-facing artifacts: A35 finite-grid frontier, A36 funded-set grade - audit, A37 selected-allocation tail-risk repricing, A38 cluster-bound audit, - and A39 fixed-allocation bootstrap diagnostic. - -The previous IJDS rebaseline is retained as historical provenance, not as the -active body claim: +## Active paper contract + +The current IJDS body point is the simple calibration-selected 90% guardrail: + +- run tag: + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` +- exact conformal replay: target `alpha=0.10`, frozen used alpha `0.095`; +- policy: `q=(p+u)/2`, `tau=0.17`, with point PD in the economic objective + and conformal `q` in the risk constraint; +- selector: nine round-number policies on November 2017, five eligible under + deterministic full-budget, effective-PD, and `B_u<=0.28` screens; no outcome + or assumption-conditional selector columns; +- independent audit: the outcome-free December replay selects the same policy; + after outcomes are opened, weighted default is `0.145650` and miscoverage is + `0.124925`, explicitly ruling out a selected-set validity claim; +- realized return: `$179,327.59`; +- weighted default and miscoverage: `0.039375` and `0.036875`; +- `Gamma_CP=0.176102`, `Gamma_residual=0.088051`; +- endpoint budget `0.258051`; assumption-conditional Markov threshold + `0.574279`; +- paper artifacts: A35 exact-alpha audit, A36 split selector/audit, A37 + temporal evaluation, A38 grade audit, A39 month-cluster bootstrap with + loan-level sensitivity, and A40 matched comparisons. + +The selector is outcome-free with respect to OOT policy ranking, but earlier +project development inspected the static OOT corpus. The paper must describe +the result as a retrospective lockbox replay, not a pristine prospective trial. + +The previous IJDS rebaseline and pool93 frontier are retained as historical +provenance, not as active body claims: - run tag: `ijds-rebaseline-2026-06-07` - policy: `bound_aware_276k_economic_champion` @@ -70,7 +80,10 @@ Do not overwrite these protected files without an explicit revalidation plan: - `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` - `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` - `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` +- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` - `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` - `EXTRACTION_MANIFEST.json` diff --git a/docs/refactor/README.md b/docs/refactor/README.md index da21078..b8d70ab 100644 --- a/docs/refactor/README.md +++ b/docs/refactor/README.md @@ -21,6 +21,7 @@ Each plan documents: | [`CONFORMAL_REFACTOR_PLAN.md`](CONFORMAL_REFACTOR_PLAN.md) | Yes (calibrator pickle) | Full public split executed 2026-06-13; `src.models.conformal` is now a package facade with strict-typed submodules. | | [`MAPIE_MIGRATION_PLAN.md`](MAPIE_MIGRATION_PLAN.md) | Yes (intervals parquet) | Runtime is already MAPIE 1.x and the drift report is green; protected reruns still require explicit approval. | | [`archive/FEATURE_CONFIG_PARQUET_PLAN.md`](archive/FEATURE_CONFIG_PARQUET_PLAN.md) | Yes (downstream stages) | Executed 2026-06-13 and archived; `feature_config.pkl` retired from the live DVC DAG and manifest. | +| [`ijds_tooling_decisions_2026-07-09.md`](ijds_tooling_decisions_2026-07-09.md) | No (tooling/refactor only) | Final live contract: `uv`, Ruff, mypy, `ty`, pytest, `just`, DVC and hook validation; Pyrefly and Commitizen are not adopted. | Executed lanes now in `main`: diff --git a/docs/refactor/ijds_tooling_decisions_2026-07-09.md b/docs/refactor/ijds_tooling_decisions_2026-07-09.md new file mode 100644 index 0000000..c703407 --- /dev/null +++ b/docs/refactor/ijds_tooling_decisions_2026-07-09.md @@ -0,0 +1,127 @@ +# IJDS Tooling and Refactor Decisions - 2026-07-09 + +This is the final decision record for the IJDS code and manuscript workflow. +It replaces the iterative tooling lab from 2026-07-08, whose useful decisions +have been implemented. Scientific history remains in `docs/research/`; this +file describes only the live engineering contract. + +## Objective + +Keep one auditable route from frozen upstream artifacts to the submitted +claim. Prefer small, explicit modules and named commands over additional +frameworks, parallel implementations, or hidden manuscript-time computation. + +## Adopted tools + +| Tool | Role | Decision | +|---|---|---| +| `uv` | Python environment, lockfile, commands | Sole Python package/runtime interface. | +| Ruff | lint and formatting | Sole linter and formatter. | +| mypy | stable gradual type gate | Required by `just type-check`. | +| ty | fast independent type audit | Active and full scopes; full scope blocks submission closeout. | +| pytest | behavioral and claim-sync tests | Required for focused, smoke, and full suites. | +| just | named local workflow | Sole human-facing command menu. | +| DVC | frozen artifact lineage | Keep for scientific provenance; never use it as a general task runner. | +| pre-commit + prek | hook compatibility and fast config validation | Keep both checks; do not create a second hook policy. | +| pdoc | optional local API browsing | Ephemeral via `uv run --with pdoc`; no project dependency. | + +`ty` complements rather than replaces mypy. The active scope includes the +exact-alpha replay and calibration-selected policy modules. Its full clean +scope is useful as an independent submission check, while mypy remains the +stable repository contract. + +## Rejected additions + +| Tool | Decision | Reason | +|---|---|---| +| Pyrefly | Do not adopt | It duplicated type checking and produced substantially more migration noise than actionable signal. | +| Commitizen | Do not adopt | Commit-message automation does not improve scientific validity or the one-author release flow. | +| Permanent pdoc dependency | Do not adopt | Generated API pages are useful locally but are not a publication artifact. | +| A second task runner | Do not adopt | `just` already exposes the complete Windows-first workflow. | +| Automatic semantic versioning | Do not adopt | Run tags, Git commits, and evidence hashes are the relevant scientific identifiers. | + +## Live methodology path + +1. `src/models/conformal_alpha_grid.py` exactly replays the frozen 90% + intervals and reports the alpha sensitivity. +2. `src/optimization/policy_evaluation.py` uses point PD in the economic + objective and an effective PD only in the risk constraint. +3. `src/optimization/policy_selection.py` defines the nine-cell round-number + grid, deterministic endpoint screen, and cap-stability interval. +4. `scripts/experiments/ijds_policy_support.py` aligns candidate and exact-alpha + rows directly by ID and owns shared solving/evaluation. +5. `scripts/build_ijds_calibration_selected_evidence.py` materializes A35-A40 + from versioned experiment outputs. Manuscript rendering does not solve or + retune portfolios. + +The active run is +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. +The exact-alpha run is +`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. + +## Deliberate simplifications + +- One active policy family: `q=(p+u)/2`, `tau=0.17`, `gamma=0.50`. +- Point PD remains the economic objective; uncertainty is a feasibility + guardrail. +- One deterministic 3x3 November selector, with outcomes isolated from a + 12-column input frame, `B_u<=0.28`, and an outcome-free December replay. +- One independent December decision audit that records the funded-set coverage + miss instead of converting it into an unsupported guarantee. +- Month-cluster bootstrap is primary; loan-level resampling is a sensitivity. +- One A35-A40 active evidence bundle. +- One body, one supplement, and one official submission TeX source. +- No nested temporal selector, effective-PD objective branch, active cap/tail + variants, or manuscript-time optimizer. +- Historical A1-A34 tables remain diagnostics and provenance, not competing + active claims. + +## Named commands + +```powershell +just ijds-evidence +just ijds-active-replay +just lint +just type-check +just type-advisory-full +just smoke +just validate-champion +just drift-gate +just test +just submission-check +``` + +`just submission-check` is the ordinary release gate and includes the full +pytest suite. `just +ijds-active-replay` is intentionally separate because it recomputes exact +interval grids and solves experiment portfolios. + +## Compilation contract + +The official build uses `latexmk`. On Windows it resolves TinyTeX's +`latexmk.pl` and launches it with Perl, bypassing the defective +`runscript.tlu` executable wrapper. If that payload is unavailable or fails, +the robust fallback is: + +```text +pdflatex -> bibtex -> pdflatex -> pdflatex +``` + +The first pass writes citation keys to `.aux`, BibTeX writes `.bbl`, and the +last two passes stabilize citations, cross-references, and pagination. This is +one compilation workflow, not three independent builds. + +## Drift policy + +Experimental drift is allowed only under a new run tag. Promotion requires: + +- no writes to manifest-listed or protected champion artifacts; +- an explicit scientific reason and comparator; +- claim-sync and publication-integrity tests; +- `just validate-champion`; +- `just drift-gate` when PD or conformal paths change; +- regenerated tables and a visually inspected PDF. + +Light drift is not itself a benefit. A challenger is promoted only when it +improves the submitted method or its defensibility enough to justify the extra +surface area. Otherwise it is removed from the live path. diff --git a/docs/research/README.md b/docs/research/README.md index a3bbdc5..567aa29 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -9,9 +9,11 @@ perenne y lo que el código lee/escribe. ## Registros activos (referenciados por el código o el paper) - `active_claims_2026-07-04.md` — source-of-truth operativo del claim IJDS - activo: dual-tag governance, punto pool93, semántica de grilla finita, - denominadores de frontera y stop rules para no reabrir búsquedas sin un - claim que pueda cambiar. + activo: replay conformal exacto al 90%, selector de calibración 3x3, + política lineal 50/50, evidencia A35--A40 y stop rules. +- `ijds_exact_alpha_calibration_selection_2026-07-09.md` — closeout que + documenta por qué se retiró el alpha-0.01 aproximado y cómo se eligió la + política simple sin outcomes OOT en el selector final. - `crpto_p1_evidence_2026-05-04.md` — evidencia P1 alrededor del champion congelado (escrito por `scripts/analyze_crpto_evidence.py`). - `crpto_journal_package_2026-05-04.md` — tablas A12–A34 y figuras journal @@ -40,8 +42,19 @@ perenne y lo que el código lee/escribe. limpieza local de peso y frontera entre refactor estricto y nueva corrida tolerante. - `ijds_scientific_upgrade_audit_2026-07-07.md` — separación entre mejoras de - paper que pueden entrar con evidencia congelada y extensiones que requieren - un nuevo resultado CRPTO v2. + paper que pueden entrar con evidencia congelada y extensiones que quedan + fuera del claim enviado salvo un nuevo protocolo etiquetado. + +- `ijds_corpus_claims_improvement_plan_2026-07-07.md` - analisis con + `academic-pdf-intake` del paper, supplement, submission PDF y corpus + `Papers_tesis`; sus recomendaciones editoriales IJDS quedaron aplicadas y se + conserva como trazabilidad, no como backlog abierto. +- `ijds_literature_expansion_scan_2026-07-08.md` - scan de literatura externa y + local para nuevas referencias IJDS: contextual optimization, incertidumbre de + credit scoring, conformal no-exchangeable, post-selection y comparadores + decision-calibrated 2026. +- `pool93_certificate_semantics_v2_2026-07-09.md` - auditoría histórica de la + frontera pool93 y su baseline; conserva procedencia, no claims activos. ## Registros de gobernanza (decisiones; no se re-ejecutan sin permiso) diff --git a/docs/research/active_claims_2026-07-04.md b/docs/research/active_claims_2026-07-04.md index c22ef66..f765e9f 100644 --- a/docs/research/active_claims_2026-07-04.md +++ b/docs/research/active_claims_2026-07-04.md @@ -1,164 +1,168 @@ -# CRPTO Active Claim Registry - 2026-07-04 - -This registry is the current source of truth for paper-facing CRPTO claims. It -supersedes older research notes that centered the `45/45` local region or the -`paper-thesis-final-economic-2026-04-06` run as the active manuscript result. -Those notes remain useful as provenance, not as active operating instructions. - -## Active Body Claim - -CRPTO is an auditable conformal-robust credit-portfolio decision certificate: -a frozen calibrated PD model feeds Mondrian conformal upper endpoints, an -exact finite policy-grid portfolio search exposes a return-bound frontier, and -the selected funded set is audited on the full OOT universe. - -Current body/default point: - -- terminal run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` -- body point source run: - `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` -- policy family: `claim_micro_ext_body_cap345` -- policy mode: `capped_blended_uncertainty` -- risk tolerance: `0.1715` -- gamma: `0.5475` -- delta cap quantile: `0.975` -- uncertainty aversion: `0.05` -- realized return on a 1M budget: `$184,832.48` -- return-floor surplus: `$14,367.94` -- `V(alpha=0.01)`: `0.035350` -- `Gamma_CP(alpha=0.01)`: `0.162616` -- endpoint budget upper at `alpha=0.01`: `0.24508374` -- Markov cap at `alpha=0.01`: `0.34508374` -- exact alpha violation: `0.0` -- declared alpha-grid pass: `8/8` -- fixed-allocation bootstrap return interval: - `$167,963.20`--`$198,650.47` - -Do not describe this as a newly retrained champion. The active pool93 claim is a -deterministic policy-grid re-evaluation over the same frozen upstream PD model, -calibrator and conformal interval outputs. - -## Active Evidence - -| Claim | Decision | Destination | Evidence | Stop Rule | -|---|---|---|---|---| -| CRPTO is a decision certificate, not a classifier leaderboard. | Promote | body | `paper/CRPTO_ijds.qmd`, Figure 1, exact certificate table, A35 | Do not reopen unless the decision certificate changes. | -| The pool93 body point is selected from a finite exact return-bound frontier. | Promote | body + A35 | `crpto_tableA35_pool93_ijds_frontier.csv`, consolidated governance JSON | Do not run more portfolio search unless a new result can lower cap at same return or materially lift return under the declared cap. | -| The selected allocation has inspectable business composition and tail profile. | Append | supplement A36--A39 | A36 grade audit, A37 LGD/CVaR/OCE repricing, A38 cluster-bound audit, A39 bootstrap | Diagnostics only; do not use as hidden selector. | -| The former `45/45` rebaseline remains provenance and return floor. | Archive/Append | provenance/supplement | `EXTRACTION_MANIFEST.json`, `ijds_rebaseline_2026-06-07.md` | Do not use as active headline except to explain the declared floor. | -| External Prosper/Freddie runs support recipe transfer. | Append | body short paragraph + supplement A25--A34 | external replication tables and figures | Do not promote as new Lending Club certificates. | - -## Finite-Grid Semantics - -The declared alpha grid is: - +# CRPTO Active Claim Registry - 2026-07-09 + +This file is the source of truth for IJDS-facing claims. The active result is a +simple, calibration-selected 90% conformal guardrail. Older pool93 frontier +files remain immutable provenance under `EXTRACTION_MANIFEST.json`; they are no +longer evidence for the manuscript's main claim. + +## Active Decision + +- Run tag: + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` +- Conformal target: `alpha = 0.10`; frozen conservative alpha used by the + recipe: `0.095`. +- Partition: five score-quantile Mondrian cells on calibrated PD (the frozen + artifact retains the historical label `score_decile_mondrian`). +- Decision score: `q_i = p_i + 0.50 (u_i - p_i) = (p_i + u_i) / 2`. +- Portfolio risk tolerance: `tau = 0.17`. +- Economic objective: expected point-PD net return, `c_i - p_i L`, with + `L = 0.45`. Conformal `q_i` enters the risk constraint, not the objective. +- Budget: `$1,000,000`; maximum concentration: `0.25`. + +The policy is selected from nine round-number candidates: +`tau in {0.15, 0.17, 0.19}` crossed with +`gamma in {0.25, 0.50, 0.75}`. The final tagged selector uses November 2017, +requires full budget use, enforces the effective-PD cap and deterministic +`B_u <= 0.28`, and maximizes expected point-PD objective. Five of nine +candidates are eligible; the selected policy is `tau = 0.17, gamma = 0.50`. +The selected row remains optimal for endpoint caps in +`[0.259036, 0.290491)`. + +The conformal recipe uses `142,550` calibration-fit rows. Policy selection is +performed on `14,943` November rows. Outcomes are stored separately from the +12-column selector frame, which contains no defaults, realized returns, +miscoverage, or assumption-conditional fields. An outcome-free replay on +`20,695` December rows independently selects the same policy. Outcomes joined +afterward give weighted default `0.145650`, weighted +miscoverage `0.124925`, endpoint budget `0.262082`, and accounting bound +`0.387007`. This audit is diagnostic evidence that stable policy selection is +not selected-set coverage validity. Conformal endpoints themselves use +calibration labels, as required. + +## Full OOT Result + +The fixed policy is evaluated on `276,869` loans from January 2018 through +September 2020: + +| Quantity | Value | +|---|---:| +| Funded loans | `308` | +| Allocated budget | `$1,000,000` | +| Expected point-PD objective | `$168,271.56` | +| Realized return | `$179,327.59` | +| Weighted default rate | `0.039375` | +| Weighted miscoverage `V` | `0.036875` | +| Weighted point PD | `0.081949` | +| Weighted decision score | `0.170000` | +| `Gamma_CP` | `0.176102` | +| `Gamma_internalized` | `0.088051` | +| `Gamma_residual` | `0.088051` | +| Endpoint budget `B_u` | `0.258051` | +| Observed accounting bound `B_u + V` | `0.294926` | +| Markov event threshold `B_u + sqrt(alpha)` | `0.574279` | + +The primary fixed-allocation bootstrap return interval is +`$163,421.14`--`$193,551.65` (`5,000` draws over `31` origination-month +clusters). A funded-loan sensitivity gives +`$162,706.17`--`$193,924.74`. Neither resamples the model, conformal recipe, +selector, or optimizer. + +## Matched Baseline + +The point-PD comparator uses the same `276,869` candidates, budget, +concentration cap, `tau = 0.17`, LGD, solver, and point-PD economic objective. +It earns `$196,369.14`, funds `225` loans, and has weighted default `0.118400`, +miscoverage `0.041900`, endpoint budget `0.921317`, and Markov threshold +`1.237545`. + +Relative to that comparator, selected CRPTO gives up `$17,041.55` (`8.678%`) +of realized return, reduces weighted default by `7.9025` percentage points, +reduces weighted miscoverage by `0.5025` percentage points, and lowers the +endpoint-plus-Markov threshold by `66.3266` percentage points. These are +retrospective OOT contrasts, not causal effects or universal dominance. + +A more conservative `gamma = 0.75` comparator earns `$172,939.50`, with +weighted default `0.035875` and threshold `0.516624`. It shows the remaining +within-CRPTO trade-off: the selected 50/50 policy earns `$6,388.08` more at +`0.35` percentage points more weighted default and a `0.057655` higher +threshold. + +## Exact Alpha Evidence + +The exact replay reproduces the stored 90% reference intervals to numerical +precision (`max abs error <= 6.67e-16`). Other alpha levels are sensitivity +rows under the same frozen widening recipe, not separately selected policies. +The declared sensitivity grid is `A = {0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}`. -Therefore, the maximum possible alpha-grid pass for a single policy under the -current evidence bundle is `8/8`. A different maximum would require a new, -explicitly declared alpha grid and regenerated exact validation outputs. - -For the terminal endpoint run: - -- `n_policies = 37,068` -- `|A| = 8` -- maximum exact candidate-alpha checks: - `37,068 * 8 = 296,544` -- observed completion: `296,544/296,544` -- all-alpha passers: `37,068/37,068` -- all-alpha passers above the declared return floor: `14,814/37,068` - -For the consolidated frontier: - -- raw rows: `51,678` -- duplicate semantic rows removed: `1,668` -- maximum deduplicated semantic policies in this consolidated evidence file: - `50,010` -- eligible all-alpha above-floor policies: `27,508/50,010` -- nonpass or below-floor policies: `22,502/50,010` - -These denominators are finite-grid denominators, not continuous optimality -claims. If future work adds new policy families, gamma values, alpha levels or -solvers, the denominators can grow under a new run tag; they must not be mixed -with the current frozen denominators. - -## How To Present The Denominators - -The result should be presented as a finite-grid decision certificate, using the -same style that robust optimization and conformal risk-control papers use for -tunable risk levels: state the declared grid, report the denominator, expose the -trade-off frontier, and separate the selected body point from endpoints. - -Recommended body wording: - -> The selected policy is chosen from a declared finite policy-grid frontier. The -> consolidated frontier contains 50,010 deduplicated semantic policies, of which -> 27,508 both pass every declared alpha level and exceed the return floor. The -> terminal exact endpoint search evaluates 37,068 policies across eight alpha -> levels, for 296,544 candidate-alpha checks, and all terminal policies pass the -> all-alpha audit. These counts certify the declared finite search surface; they -> are not a continuous global-optimality claim. - -Use `pool93` only for run tags, file names, governance JSONs, and internal -provenance. The body manuscript should say selected policy, selected decision, -or declared finite-grid frontier unless the run label is needed to disambiguate -an evidence source. - -Report the screenshot numbers with explicit denominators: - -- `alpha_grid_pass = 8/8`: one selected policy passed all eight declared alpha - levels. The current maximum is eight because the alpha grid has eight - levels. -- `50,010` semantic policies: the maximum deduplicated policy denominator in - the current consolidated frontier. It equals `51,678` raw rows minus `1,668` - duplicate semantic policies. -- `27,508` eligible all-alpha above-floor policies: the number of consolidated - semantic policies that satisfy both gates: all-alpha pass and nonnegative - surplus over the declared return floor. -- `37,068/37,068` terminal all-alpha passers: every policy in the terminal - endpoint search passed all eight declared alpha checks. -- `296,544/296,544` exact terminal checks: the run completed all - `37,068 * 8` policy-alpha evaluations. This is primarily a completion - denominator; because all 37,068 policies are all-alpha passers, it also means - there were no failed alpha cells inside the terminal surface. - -Do not present these as: - -- proof over all possible policy hyperparameters; -- proof over all possible alpha values in `(0, 1)`; -- a live-production coverage guarantee after adaptive policy selection; -- evidence that more policies are always better. - -The IJDS framing should emphasize data + methodology + decision + implication: -the finite-grid frontier is the decision object, the exact checks are the -auditable computation, and the endpoints expose the price of robustness. +Do not reinstate the former `alpha = 0.01` headline. Its exact intervals have +average width `0.9882`, and `93.54%` of OOT upper endpoints equal one. That +setting is nearly vacuous for portfolio discrimination. The active method uses +the conventional 90% level because it is the recipe's selected and exactly +replayed reference level. ## Theory Boundary -The paper-facing theorem uses deterministic accounting plus a distribution-free -Markov step under weighted funded-set validity. The body keeps Markov because -it is the weakest defensible assumption for the current selected allocation. -A38 reports cluster-aware thresholds as sensitivity; none is tighter than -Markov for the observed exposure concentration. +For a fixed funded set, the deterministic accounting identity + +`weighted outcome <= B_u + V` + +holds without a statistical assumption. In the active OOT allocation its +right-hand side is `0.294926`, while the observed weighted outcome is +`0.039375`. + +The Markov statement is secondary. If one additionally assumes weighted +funded-set validity, `E[V] <= alpha`, then + +`P(weighted outcome >= B_u + sqrt(alpha)) <= sqrt(alpha)`. + +At `alpha = 0.10`, this gives threshold `0.574279` and probability bound +`0.316228`. It is deliberately reported as a weak, assumption-conditional +sensitivity, not as a deterministic risk cap, nominal selected-set coverage, +or the paper's primary novelty. + +## Evidence Contract + +The active evidence bundle is: + +- `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` +- A35: exact alpha replay and saturation audit. +- A36: November selector, endpoint-cap stability interval, December replay, + and independent decision audit. +- A37: full-OOT and temporal fixed-policy evaluation. +- A38: selected funded-set grade composition. +- A39: fixed-allocation month-cluster bootstrap plus funded-loan sensitivity. +- A40: selected, more-conservative, and matched point-PD comparison. + +The manuscript must say explicitly that earlier project development inspected +this static OOT corpus. The final tagged rule is outcome-free with respect to +its policy-ranking code path, but the evaluation is a transparent retrospective +lockbox replay, not a pristine prospective trial. + +## Retired Headline Claims + +The following remain provenance only and must not appear as active results: -Do not claim: +- alpha-0.01 endpoints obtained by cross-family average-width scaling; +- the `8/8` approximate alpha-grid pass; +- the 50,010-policy frontier as the active selector; +- the `0.345084` Markov threshold; +- capped/tail-focused policy families as the selected method; +- the exploratory 25-policy `gamma = 0.35`, threshold-cap `0.65` challenger; +- policy hyperparameters chosen from OOT realized outcomes. -- universal conditional coverage; -- global optimum over continuous policy parameters; -- future live-deployment validity; -- a CVaR/OCE/bootstrap-selected champion; -- that `8/8` is an external standard rather than the declared grid. +The frozen upstream model, calibrator, interval artifacts, historical pool93 +tables, and `EXTRACTION_MANIFEST.json` remain untouched. ## Reopen Gate -A new search is justified only if it can plausibly change one of these claims: +Reopen the active method only for one of four reasons: -1. same or higher return with materially lower Markov cap or `Gamma_CP`; -2. much higher return under the same declared cap; -3. a denser predeclared alpha grid that materially strengthens the certificate; -4. a nested/prospective evaluation design that reduces post-selection risk; -5. a reviewer-requested diagnostic that closes a specific objection. +1. a calibration-only rule materially improves return at the same `B_u<=0.28` + screen; +2. a simpler rule matches the selected policy within prespecified tolerances; +3. a valid selected-set or prospective protocol materially strengthens the + statistical claim; +4. an IJDS reviewer requests a specific additional test. -Otherwise, append the idea to future work and keep the current pool93 frontier -closed. +Otherwise, keep one method, one policy, and one manuscript narrative. diff --git a/docs/research/crpto_editorial_claims_references.qmd b/docs/research/crpto_editorial_claims_references.qmd index ee2a516..a69c736 100644 --- a/docs/research/crpto_editorial_claims_references.qmd +++ b/docs/research/crpto_editorial_claims_references.qmd @@ -1,10 +1,11 @@ ## Guía Editorial, Claims y Referencias {#sec-crpto-editorial-guide} -> Nota histórica, 2026-07-04: esta guía fue escrita antes del cierre pool93. +> Nota histórica, actualizada 2026-07-09: esta guía fue escrita antes del cierre pool93. > La fuente activa de métricas y límites de claim es > `docs/research/active_claims_2026-07-04.md`. Las referencias a > `paper-thesis-final-economic-2026-04-06`, `45/45` o A7--A11 se mantienen como -> provenance editorial; el cuerpo IJDS actual usa el punto pool93 y A35--A39. +> provenance editorial; el cuerpo IJDS actual usa el punto pool93, la frontera +> policy-aware A35, las auditorías A36--A39 y el baseline emparejado A40. Esta página existe porque el libro cumple una función distinta al paper. El paper final debe ser breve, selectivo y persuasivo; el libro puede ser más @@ -32,7 +33,7 @@ La frase anterior tiene cuatro piezas: riesgo [1]--[5]; 2. **restricción operativa**: conecta esos intervalos con robust optimization y price of robustness [6]--[9]; -3. **policy económica promovida**: usa el cierre pool93 documentado en A35--A39 +3. **policy económica promovida**: usa el cierre pool93 documentado en A35--A40 y no reabre la cadena upstream congelada; 4. **cautela post-selección**: separa el teorema Markov del tightening condicional y de las validaciones empíricas A7--A11. @@ -50,8 +51,8 @@ empíricos, otros son de ingeniería reproducible. | 3 | El intervalo puede convertirse en conjunto de incertidumbre | definición de `u_i(alpha)` y `Gamma_CP` | no tratar `Gamma_CP` como presupuesto ad hoc | | 4 | El bound controla no-cobertura ponderada del funded set | `thm-conformal-feasibility` y `V` | no decir que controla directamente una PD latente sin supuesto adicional | | 5 | Existe una frontera robusta finita en OOT | A35: `50,010` políticas semánticas deduplicadas y `27,508` elegibles all-alpha sobre piso; terminal `37,068/37,068` all-alpha passers | no decir que eso certifica una región continua ni un óptimo global | -| 6 | El body point pool93 es económico, no theorem-tight | retorno `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084` | no mezclarlo con el endpoint mínimo-cap ni con el max-return endpoint | -| 7 | La evidencia CRPTO fortalece el paper sin cambiar dirección | A35--A39, apéndice condicional, A25--A34 | no venderlo como validación live o universal | +| 6 | El body point pool93 es económico, no theorem-tight | retorno `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, umbral exacto de pérdida `0.345084` | no mezclarlo con el endpoint de menor umbral ni con el max-return endpoint | +| 7 | La evidencia CRPTO fortalece el paper sin cambiar dirección | A35--A40, apéndice condicional, A25--A34 | no venderlo como validación live, causal o universal | : Escalera editorial de claims del CRPTO {#tbl-crpto-claim-ladder} @@ -112,7 +113,9 @@ bien confirma que la dirección del paper es contemporánea y fértil. | `alpha` | nivel de tolerancia a no-cobertura | cuánto riesgo de cola acepta el comité | | `PD_low`, `PD_high` | intervalo conformal por préstamo | rango defendible de riesgo | | `u_i(alpha)` | cota superior usada por el LP | PD prudente para decisión | -| `Gamma_CP` | prima conformal ponderada | costo promedio de comprar robustez | +| `Gamma_CP` | ancho conformal ponderado total | incertidumbre disponible antes de fijar la policy | +| `Gamma_int` | intervención prudencial usada por la policy | robustez efectivamente comprada | +| `Gamma_res` | ancho conformal residual | holgura que completa el certificado policy-aware | | `V` | no-cobertura ponderada del funded set | cuánto default realizado quedó fuera de la cota | | `violation` | exceso de riesgo ponderado sobre `tau` | incumplimiento del límite de portafolio | | `alpha01_exact_pass` | check exacto de bound en OOT | semáforo para promoción paper-facing | @@ -188,7 +191,7 @@ paper-grade, incluida la nueva Figura 1 IJDS, generadas o publicadas desde artef | Blueprint del manuscrito | estructura paper-ready y mapa claim -> artifact -> test -> location | `14g-manuscript-blueprint.qmd` | | Figura CRPTO limpia | Figura 1 IJDS | `crpto_fig1_journal_pipeline.png` | | Figura alpha -> `Gamma_CP` -> funded set | puente teoria-metodo | `crpto_fig13_alpha_gamma_funded_set.png` | -| A35 frontier y A36--A39 pool93 | evidencia visual/tabular del cierre pool93 | `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv`; `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv`; `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv`; `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv`; `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` | +| A35 frontier, A36--A39 pool93 y A40 point-PD | evidencia visual/tabular del cierre pool93 y costo comparable de robustez | `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv`; `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv`; `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv`; `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv`; `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv`; `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` | | A12--A34 | robustness appendix journal | `scripts/build_crpto_journal_package.py`; `scripts/build_tail_satisficing_challenger_audit.py`; `scripts/build_tail_constrained_reoptimization.py`; `scripts/build_distribution_robustness_diagnostics.py`; `scripts/build_multidataset_external_replication.py`; `scripts/build_price_of_robustness_cross_dataset.py` | | A25--A34 | réplica externa Prosper/Freddie | `models/crpto_multidataset_external_status.json` | | Status del paquete | trazabilidad y fuente canonica | `models/crpto_journal_package_status.json`; `models/crpto_multidataset_external_status.json` | diff --git a/docs/research/crpto_full_audit_2026-07-05.md b/docs/research/crpto_full_audit_2026-07-05.md index 8d237fa..e041918 100644 --- a/docs/research/crpto_full_audit_2026-07-05.md +++ b/docs/research/crpto_full_audit_2026-07-05.md @@ -1,5 +1,11 @@ # Auditoría integral CRPTO — 2026-07-05 +> **Cierre semántico posterior (2026-07-09).** Las secciones de proceso y +> refactor de este memo conservan valor histórico. Para métricas del +> certificado, frontera A35 y baseline A40, prevalece +> `pool93_certificate_semantics_v2_2026-07-09.md`; ese cierre reemplaza el +> atajo lineal que este memo llamaba "Markov cap". + Auditoría de consistencia de claims, parsimonia de código, skills, docs y plan editorial del paper, de cara a la submission IJDS (target interno 2026-08-10). Método: tres pasadas de exploración paralelas (código, claims, tooling/docs) @@ -38,9 +44,9 @@ canónica: `docs/research/active_claims_2026-07-04.md`. | Retorno body point pool93 | $184,832.48 | OK en qmd, tex (`\$184{,}832.48`), A35, gobernanza | | V(alpha=0.01) | 0.035350 | OK en todas las fuentes | | Gamma_CP(alpha=0.01) | 0.162616 | OK; notación Gamma_CP vs gamma de política bien separada | -| Markov cap (alpha=0.01) | 0.345084 | OK (A35 guarda 0.345083740; redondeo esperado) | -| Endpoint budget upper B_u | 0.245084 = 0.1715 + 0.4525 * 0.162616 | OK en body, supplement y A35 | -| Alpha grid | 8/8, violación exacta 0.0 | OK | +| Umbral exacto de pérdida (alpha=0.01) | 0.345084 | OK en la A35 policy-aware corregida | +| Endpoint budget upper B_u | 0.245084 = sum(w q) + Gamma_res | OK; en el body point la cota por fila está inactiva y el atajo lineal coincide numéricamente | +| Alpha grid | 8/8 screen pass; exceso realizado sobre tau 0.0 | OK | | Return floor declarado | $170,464.54 | OK; en gobernanza y supplement, no como headline | | Frontera consolidada | 50,010 semánticas dedup; 27,508 all-alpha sobre floor | OK | | Búsqueda terminal | 37,068/37,068 passers; 296,544 checks | OK | diff --git a/docs/research/crpto_publication_strategy_2026-05-12.md b/docs/research/crpto_publication_strategy_2026-05-12.md index c275f90..c13e3ff 100644 --- a/docs/research/crpto_publication_strategy_2026-05-12.md +++ b/docs/research/crpto_publication_strategy_2026-05-12.md @@ -66,7 +66,7 @@ The first paper draft should be written as: - title: `CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit Portfolio Decisions`; - body: 25-page IJDS-style manuscript; -- supplement: A3--A39, proofs, extended tables, reproducibility, +- supplement: A3--A40, proofs, extended tables, reproducibility, MRM/fairness and external replication; - review mode: anonymous by default; - companion: GitHub/DVC/DagsHub/MLflow after the anonymity policy is handled. @@ -77,8 +77,8 @@ The short paper should keep only the strongest body material: - PD calibration summary; - Mondrian conformal layer; - robust portfolio formulation; -- Markov bound and conditional tightening caveat; -- pool93 body metrics and A35 finite-grid frontier; +- policy-aware decision certificate and conditional tightening caveat; +- pool93 body metrics, A35 finite-grid frontier and A40 matched point-PD audit; - A36--A39 selected-allocation audits; - regret-auditability frontier with one concise SPO+/DFL comparison; - data/code reproducibility statement. @@ -100,13 +100,13 @@ In scope for the current paper: - promoted pool93 finite-grid return-bound frontier; - frozen upstream CRPTO chain as provenance and return floor; - calibrated PD -> Mondrian conformal intervals -> robust portfolio decision; -- exact alpha-safe funded-set validation and A35 finite-grid frontier; -- A3--A39 as supplement evidence; +- exact funded-set validation and A35 policy-aware finite-grid frontier; +- A3--A40 as supplement evidence; - regret-auditability frontier in the body; - OCE/CVaR tail-risk diagnostics and robust satisficing margins in the supplement; -- cluster-aware dependence caveat/proposition with Markov retained as the main - distribution-free bound; +- cluster-aware dependence caveat/proposition with the Markov step retained in + the main distribution-free decision certificate; - external economic replication on Prosper and Freddie/Mendeley as A25--A34, without reopening the Lending Club body claim; - reproducibility via Quarto, DVC, DagsHub/MLflow and guardrail tests. diff --git a/docs/research/ijds_claim_concept_audit_2026-06-26.md b/docs/research/ijds_claim_concept_audit_2026-06-26.md index 7c46ae2..2258b59 100644 --- a/docs/research/ijds_claim_concept_audit_2026-06-26.md +++ b/docs/research/ijds_claim_concept_audit_2026-06-26.md @@ -1,5 +1,10 @@ # IJDS Claim Concept Audit: Alpha Grid, Robust Region, Bound, and Exact Frontier +> **Historical only.** This pre-closeout analysis uses the former linear +> certificate shortcut. Current claims and corrected endpoint thresholds live +> in `active_claims_2026-07-04.md` and +> `pool93_certificate_semantics_v2_2026-07-09.md`. + Date: 2026-06-26 This memo audits the concepts that are easy to take for granted in the CRPTO diff --git a/docs/research/ijds_claim_maximization_analysis_2026-06-27.md b/docs/research/ijds_claim_maximization_analysis_2026-06-27.md index 1507e99..6002fc4 100644 --- a/docs/research/ijds_claim_maximization_analysis_2026-06-27.md +++ b/docs/research/ijds_claim_maximization_analysis_2026-06-27.md @@ -1,5 +1,9 @@ # IJDS Claim Maximization Analysis - 2026-06-27 +> **Historical only.** This search diary predates the policy-aware certificate +> correction. Do not reuse its cap labels or endpoint thresholds; current A35 +> and A40 evidence is registered in `pool93_certificate_semantics_v2_2026-07-09.md`. + This memo evaluates the strongest IJDS-facing claims for CRPTO after the pool93 claim-governance and local exact-refinement work. It is an internal research artifact, not manuscript prose. diff --git a/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md b/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md new file mode 100644 index 0000000..0333702 --- /dev/null +++ b/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md @@ -0,0 +1,643 @@ +# IJDS Corpus, Claims, and Improvement Plan - 2026-07-07 + +> **Superseded metric vocabulary (2026-07-09).** This memo is retained as the +> corpus-reading and editorial decision trail. Its old "Markov cap" and +> preliminary frontier language must not be used as current evidence. The +> policy-aware A35 correction and matched A40 baseline in +> `pool93_certificate_semantics_v2_2026-07-09.md` are authoritative. + +Scope: analyze the current CRPTO body paper, official IJDS submission PDF/source, +online supplement, frozen metrics/artifacts, and the local `Papers_tesis` corpus +using the global `academic-pdf-intake` skill outputs. + +This memo does **not** reopen the champion, does **not** modify +`EXTRACTION_MANIFEST.json`, and does **not** recommend rerunning protected DVC +stages. The active paper claim remains the finite-grid decision certificate +registered in `docs/research/active_claims_2026-07-04.md`. + +## Implementation Status + +Status on 2026-07-07: the actionable P0/P1/P2 editorial recommendations in this +memo have been applied to the manuscript, official IJDS `.tex` handoff, +supplement, submission checklist, bibliography, and reviewer-defense matrix. +This file is retained as traceability for the corpus/IJDS analysis and parser +evidence, not as an open TODO list. Future edits should use +`paper/submission/SCHOLARONE_FINAL_CHECKLIST.md` and +`docs/research/active_claims_2026-07-04.md` as the active operating gates. + +## Inputs Used + +Skill / benchmark outputs: + +- Full local inventory: + `.tmp_pdf_intake_benchmark/run_20260707_1715/manifest.jsonl` +- Full parser benchmark: + `.tmp_pdf_intake_benchmark/run_20260707_1715/runs.jsonl` +- Current literature matrix generated from extracted text: + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/corpus_current_inventory.csv` +- IJDS venue snippets: + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/ijds_venue_snippets.csv` +- Parser summary for active CRPTO PDFs: + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/active_crpto_parser_benchmark.csv` +- MinerU CUDA follow-up for active CRPTO PDFs: + `.tmp_pdf_intake_benchmark/run_20260707_active_mineru_cuda/runs.jsonl` + +CRPTO sources: + +- `paper/CRPTO_ijds.qmd` +- `paper/submission/CRPTO_ijds_submission.tex` +- `paper/submission/CRPTO_ijds_submission.pdf` +- `paper/supplement_ijds.qmd` +- `paper/CRPTO_ijds.pdf` +- `paper/supplement_ijds.pdf` +- `reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv` +- `reports/crpto/tables/crpto_tableA25_external_replication_gate.csv` +- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` +- `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv` +- `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` +- `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` +- `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` +- `models/experiments/champion_reopen/.../pool93_ijds_claim_governance.json` +- `models/experiments/champion_reopen/.../pool93_ijds_consolidated_governance.json` + +IJDS public sources checked on 2026-07-07: + +- Submission guidelines: + +- Reviewer guidelines: + +- Editorial statement: + +- Data and Code Disclosure Policy: + + +## Executive Diagnosis + +CRPTO is well matched to IJDS when written as **data science for decisions**: +real credit data, a methodological bridge from calibrated prediction to robust +optimization, managerial/model-risk relevance, practical implications, and +reproducible computational evidence. The strongest submission story is not +"better credit scoring." It is: + +> CRPTO converts a frozen calibrated PD model into an auditable robust funding +> decision by carrying conformal uncertainty into a finite-grid portfolio +> frontier, exposing the return-bound trade-off and validating the selected +> funded set from frozen evidence. + +The paper is already close to this framing. The main improvement opportunity is +reader focus: reduce the cognitive load around internal run history, make the +closest IJDS precedents visible, and make the A35 frontier the unmistakable +decision object. + +The main risk is overclaiming by shorthand. In particular, the current body and +submission use "0.345 Markov cap" as a rounded lens. The active body/default +point has `Markov_cap = 0.345083740`; the strict `<= 0.345` frontier row is a +neighboring policy with return `$184,800.41`, not the body/default return +`$184,832.48`. This is fixable in prose by saying "approximately 0.3451" or +"the declared 0.345 return-bound lens" and by avoiding exact `<= 0.345` language +for the body/default row. + +## Official IJDS Fit + +The IJDS submission guidelines say IJDS publishes data science methodologies +for decision-making environments and expects four components: data, innovative +model/algorithm/approach, managerial/engineering/industrial relevance, and +implications. CRPTO has all four: + +| IJDS component | CRPTO evidence | Current strength | Improvement | +|---|---|---|---| +| Data | Lending Club OOT panel; Prosper and Freddie/Mendeley external frozen applications. | Strong. | Keep external results as recipe transfer, not as extra certificates. | +| Model / algorithm | Calibrated PD -> Mondrian conformal intervals -> robust LP -> exact funded-set audit. | Strong. | Name the methodological unit earlier as a "decision certificate." | +| Decision relevance | `$1M` budgeted credit funding with return/risk trade-off. | Strong. | Make A35/A19 the reader-facing proof of relevance. | +| Implications | Reproducible model-risk audit surface; explicit limits on live/conditional claims. | Strong. | Put implications in abstract/conclusion as knowledge gained, not only actions performed. | + +Venue constraints that matter: + +- Initial submissions should be at most 25 IJDS-style pages excluding references + and appendices. +- IJDS uses double-anonymous review for submissions on/after 2025-01-01. +- Abstract must be one paragraph and <=300 words; IJDS strongly encourages it + to answer problem/relevance, methods/results, and insights/implications. +- Tables/figures should appear near first mention. +- Data/code disclosure is required at submission, and accepted computational + papers are expected to upload data/code or an approved alternative plan. + +The current official-template build is 26 pages total, with conclusion and +references starting on page 22, so the body appears inside the IJDS page budget +when references are excluded. The reproducibility package plan is aligned with +the IJDS policy, but review-stage path/identity sanitization remains essential. + +## Current CRPTO Claim Stack + +Active body point: + +| Quantity | Current value | +|---|---:| +| OOT Lending Club universe | `276,869` loans | +| Budget | `$1,000,000` | +| Selected-policy realized return | `$184,832.48` | +| Return-floor surplus | `$14,367.94` | +| `V(alpha=0.01)` | `0.035350` | +| `Gamma_CP(alpha=0.01)` | `0.162616` | +| Endpoint budget upper | `0.245083740` | +| Markov cap | `0.345083740` | +| Exact alpha violation | `0.0` | +| Alpha-grid pass | `8/8` | +| Bootstrap return interval | `$167,963.20` to `$198,650.47` | + +Finite-grid denominators: + +| Surface | Denominator | Result | +|---|---:|---| +| Consolidated semantic policies | `50,010` | `27,508` pass every declared alpha and exceed return floor | +| Raw consolidated rows | `51,678` | `1,668` duplicate semantic rows removed | +| Terminal endpoint policies | `37,068` | `37,068/37,068` all-alpha passers | +| Terminal alpha checks | `296,544` | `296,544/296,544` completed checks | + +Main claim boundary: + +- Claim **finite-grid decision certificate**, not continuous global optimality. +- Claim **weighted funded-set validity plus Markov bound**, not universal + conditional coverage. +- Claim **frozen external economic recipe transfer**, not Prosper/Freddie exact + Lending Club-style certificates. +- Claim **regret-auditability trade-off**, not dominance over SPO+ or other + decision-focused learners on every regret metric. +- Claim **fixed-allocation bootstrap diagnostics**, not a conformal/bootstrap + guarantee over model retraining, solver inputs, or search. + +## Paper / Submission / Supplement Assessment + +### Body and official submission + +Strengths: + +- The abstract already starts with the correct IJDS premise: credit allocation is + a data-science-for-decisions problem. +- The introduction explicitly says the research question is not a better + classifier, but whether predictive uncertainty can be carried into a robust + auditable portfolio decision. +- The contribution list separates construction, theorem, literature positioning, + frozen evidence, and evidence ladder. +- The official submission uses the IJDS `informs4` class with `dblanonrev`, and + the build instructions now document both `latexmk` and the robust + `pdflatex -> bibtex -> pdflatex -> pdflatex` fallback. +- The body has the right core tables: certificate metrics, finite-grid frontier, + reviewer-question boundary table, regret-auditability table, and supplement + map. + +Risks / repairs: + +- **Cap wording repair:** replace exact-sounding "under the declared 0.345 + Markov cap" for the body/default point with "near the declared 0.345 + return-bound lens" or "Markov cap 0.345084." Keep the strict `<=0.345` row as + a neighboring frontier endpoint, not the body point. +- **A35 should be the first durable result object.** The paper currently explains + many pieces before the reader sees the frontier. A reviewer should encounter + the frontier logic as soon as possible after the certificate table. +- **IJDS precedent paragraph is thin.** The body cites relevant work, but the + venue-specific story can be sharper: IJDS has already published credit ML, + cost-aware calibration, causal decision framing, and reproducibility-oriented + data-science-for-decisions papers. CRPTO extends that line into a portfolio + decision certificate. +- **External replications are valuable but slightly loud in the abstract.** They + support transfer of the recipe; they should not compete with the Lending Club + certificate as the abstract's main result. +- **SPO+ comparison is strong but needs a crisp takeaway sentence.** The current + text is accurate: SPO+ wins synthetic regret; CRPTO buys auditable risk + controls and a dollar-funded set. Put that exact contrast in one memorable + sentence before the table. + +### Supplement + +Strengths: + +- A35--A39 are exactly the right selected-policy closure block. +- A37--A39 correctly close tail, concentration, and empirical contribution + objections without changing the selector. +- A25--A34 answer the single-dataset concern through external economic recipe + transfer and exhaustiveness checks. +- Appendix E makes routine reproduction distinct from protected champion/search + reruns. + +Risks / repairs: + +- The supplement is rich enough that a reader may miss the hierarchy. Add a + short "how to read this supplement" map at the top that says: theory -> active + certificate -> diagnostics -> external transfer -> reproduction. +- A35 should be introduced as the supplement's active frontier, not as another + appendix table among many. +- A25--A34 should keep the words "economic replication" and "recipe transfer" + visible. Avoid "external validation" unless it is immediately scoped. +- In A38, the fact that every cluster-aware threshold is looser than Markov is a + strength: it explains why the body does not chase a more fragile bound. + +## Parser / Skill Findings + +The `academic-pdf-intake` skill routing is sensible for this repo: + +- **Docling** remains the primary parser for born-digital academic PDFs and + clean Markdown/JSON extraction. +- **OpenDataLoader hybrid** is the best comparison/fallback when bounding boxes, + reading-order traceability, hidden-text safety, and table provenance matter. +- **MinerU CUDA hybrid-engine** is now viable on the local RTX/CUDA setup for + the active CRPTO PDFs and is best kept for formula-heavy, OCR-heavy, scanned, + or visual-QA cases. +- **Codex PDF / MarkItDown** are useful as fast baselines and smoke tests, not + as the final source for complex academic extraction. + +Active PDF benchmark: + +| PDF | Pages | Docling | OpenDataLoader | MinerU CUDA | +|---|---:|---:|---:|---:| +| `paper/CRPTO_ijds.pdf` | 27 | 54.98s | 57.09s | 60.41s | +| `paper/submission/CRPTO_ijds_submission.pdf` | 26 | 46.14s | 35.00s | 54.47s | +| `paper/supplement_ijds.pdf` | 32 | 78.41s | 70.30s | 74.20s | + +Operational recommendation: + +- For day-to-day paper analysis: Docling first, ODL for table/traceability + comparison, Codex PDF for fast diffable text. +- For final figure/table/formula QA: add MinerU on the active PDF(s), then check + `layout.pdf` / visual outputs when extraction disagreement is material. +- For the full 81-PDF literature corpus: do not run all three heavy parsers + routinely. Use fast baseline + Docling on close papers, then ODL/MinerU only + on candidates with tables, equations, or layout ambiguity. + +## `Papers_tesis` Corpus Summary + +Current corpus inventory: + +- `81` local literature PDFs in `Papers_tesis` +- `3,404` literature pages +- `3` active CRPTO PDFs separately, `85` pages +- Full generated matrix: + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/corpus_current_inventory.csv` + +Topic signals from extracted text: + +| Topic signal | PDFs | +|---|---:| +| Tables / metrics / experiments | 78 | +| Tail risk / CVaR / OCE / loss | 76 | +| Portfolio / optimization / decision | 68 | +| Robust optimization / uncertainty sets | 67 | +| Conformal / coverage / calibration | 57 | +| Fairness / governance / explainability | 57 | +| Source shift / weighted / multi-source | 55 | +| Credit / lending / default | 38 | +| Causal decision | 24 | +| Decision-focused / SPO / PTO | 22 | +| Online / adaptive conformal | 20 | + +These counts are keyword/topic signals, not claims that every paper is equally +central. The editorial use is by cluster. + +## IJDS Papers in or Adjacent to the Local Corpus + +Confirmed IJDS papers with local or official evidence: + +| Paper | Evidence | What it contributes | CRPTO use | Boundary | +|---|---|---|---|---| +| Das et al. (2023), "Credit Risk Modeling with Graph Machine Learning" | Local PDF header and DOI `10.1287/ijds.2022.00018`; official IJDS page. | Extends tabular credit scoring with corporate graph features and GNN/AutoML ensembles; includes reproducibility capsule. | Shows IJDS accepts credit-risk ML when data/method/reproducibility are clear. Position CRPTO as a **decision certificate after scoring**, not a richer scorer. | Corporate credit ratings, not consumer loan portfolio funding; no conformal/robust funded-set certificate. | +| Yang and Bi (2025), "Cost-Aware Calibration of Classifiers" | Official IJDS DOI `10.1287/ijds.2024.0038`; cited in CRPTO `.bbl`. | Defines cost-aware calibration, expected calibration cost, and MetaCal; emphasizes downstream costs of miscalibration. | Strongest IJDS calibration precedent. CRPTO extends calibration into **portfolio allocation and funded-set audit**. | Classifier calibration problem, not robust portfolio optimization. | +| Fernandez-Loria and Provost (2022), "Causal Decision Making and Causal Effect Estimation Are Not the Same..." | Official IJDS DOI `10.1287/ijds.2021.0006`; cited in CRPTO `.bbl`. | Separates decision quality from effect-estimation accuracy. | Use to sharpen the intro: CRPTO is a decision object, not a prediction leaderboard. | Causal treatment assignment framing, not credit PD/conformal portfolio. | +| Fernandez-Loria and Provost (2025), "Observational vs. Experimental Data When Making Automated Decisions Using Machine Learning" | Local PDF in supplement; official DOI `10.1287/ijds.2023.0012`. | Shows observational data can sometimes support automated decisions when the decision target is ranking/thresholding rather than unbiased effect estimation. | Supports CRPTO's observational-panel boundary and the claim that decision metrics differ from estimation metrics. | Causal/automated intervention setting; CRPTO should cite it as a limitation/future protocol, not as causal validity. | +| Falconer, Kazempour, and Pinson (2026), "Toward Replication-Robust Analytics Markets" | Local PDF header and DOI `10.1287/ijds.2025.0075`; official IJDS page. | Builds an analytics market robust to strategic data replication; emphasizes reproducibility and strategic robustness. | Useful venue signal: IJDS values robust/reproducible analytics systems. Use only as a light reproducibility/robustness cousin. | Market design/collaborative analytics, not credit, conformal prediction, or portfolio funding. | + +The local corpus also references Morucci et al. (2022), an IJDS causal +uncertainty paper, but the PDF is not in `Papers_tesis`; do not count it as +local corpus evidence unless it is added. + +## Closest Non-IJDS Literature Clusters + +### 1. Conformal foundations and risk control + +Key local papers: + +- Angelopoulos and Bates (2023), gentle introduction. +- Angelopoulos et al. (2024), conformal risk control. +- Angelopoulos et al. (2025), Learn Then Test. +- Bates et al. (2021), risk-controlling prediction sets. +- Barber et al. (2021), conditional coverage limits. +- Angelopoulos et al. (2026), non-monotonic CRC. +- Gibbs/Candes, Lekeufack et al., Kiyani et al., Zhou/Orfanoudaki/Zhu. + +What they give CRPTO: + +- Validity language and finite-sample discipline. +- Justification for risk-control framing. +- Limits on conditional/group/live claims. + +How to improve paper: + +- Keep them as theory lineage, but do not over-expand. +- Use them to justify why CRPTO reports a bound and exact audit rather than only + nominal coverage. + +### 2. Conformal robust optimization / predict-then-calibrate + +Key local papers: + +- Johnstone and Cox (2021), conformal uncertainty sets for robust optimization. +- Patel et al. (2024), conformal contextual robust optimization. +- Sun et al. (2024), predict-then-calibrate. +- Zhao et al. (2026), conformal robust optimization and satisficing. +- Bao et al. (2025), CROMS model selection. +- Yeh et al. (2025/2026), conformal risk training / end-to-end calibration. +- Zhou and Zhu (2025), inverse conformal risk control. + +What they give CRPTO: + +- The nearest methodological neighborhood. +- A natural "what CRPTO adds" contrast: real credit payoff, funded-set weights, + exact portfolio audit, finite frontier, and reproducibility harness. + +How to improve paper: + +- Add a compact contrast table: abstract CRO/LP papers vs. CRPTO's credit + funded-set certificate. +- Say explicitly that CRPTO is post-hoc over a frozen PD system; end-to-end + variants are future work. + +### 3. Decision-focused learning and SPO+ + +Key local papers: + +- Elmachtoub and Grigas (2022), SPO+. +- Donti et al. (2017), task-based end-to-end learning. +- Mandi et al. (2024), DFL survey. +- Liu and Grigas (2021), risk bounds/calibration for SPO. +- Schutte et al. (2024), robust losses for DFL. + +What they give CRPTO: + +- The main alternative methodological family. +- A strong reviewer question: "Why not train through the optimizer?" + +How to improve paper: + +- Keep A19/Figure 15 central. +- State the contrast in one sentence: + "SPO+ is the low-regret corner; CRPTO is the auditable-risk-control corner + with a funded-set dollar certificate." +- Do not apologize for higher synthetic regret; explain that the metric is + different from the funded-set economic certificate. + +### 4. Credit / P2P / fairness / governance + +Key local papers: + +- Jagtiani and Lemieux (2019), fintech Lending Club context. +- Serrano-Cinca and Gutierrez-Nieto (2016), profit scoring in P2P lending. +- Guo et al. (2016), instance-based P2P credit investment. +- Zhao et al. (2016), P2P portfolio selection. +- Chi, Ding, and Peng (2019), data-driven robust P2P credit portfolio. +- Das et al. (2023), IJDS graph ML credit risk. +- Albanesi and Vamossy (2024), score performance and equity. +- Fuster et al. (2022), unequal ML credit-market effects. +- Blattner and Nelson (2021), noisy data and consumer credit disparities. +- FinRegLab (2023), explainability and fairness in credit underwriting. +- CFPB (2014), proxy race/ethnicity methods. + +What they give CRPTO: + +- Domain legitimacy and governance boundaries. +- Support for reporting economic return, risk, calibration, and governance + together. + +How to improve paper: + +- Keep fairness/proxy material bounded. CRPTO does not have protected labels or a + legal fair-lending protocol. +- Use credit/P2P papers to motivate why classification metrics alone are + insufficient for investment decisions. + +### 5. Robust optimization, DRO, tail risk, and concentration + +Key local papers: + +- Bertsimas and Sim (2004), price of robustness. +- Ben-Tal, El Ghaoui, and Nemirovski (2009), robust optimization. +- Bertsimas, Gupta, and Kallus (2018), data-driven robust optimization. +- Bertsimas and Kallus (2020), predictive to prescriptive analytics. +- Delage and Ye (2010), moment DRO. +- Goldfarb and Iyengar (2003), robust portfolios. +- Rockafellar and Uryasev (2000), CVaR. +- Ben-Tal and Teboulle (2007), OCE. +- Hoeffding, Bennett, Freedman, Fuk-Nagaev for concentration context. + +What they give CRPTO: + +- The language for price of robustness and uncertainty budgets. +- Tail-risk diagnostics and the reason to keep Markov as the weakest defensible + body-level statement. + +How to improve paper: + +- A37/A38 should be discussed as "assumption-priced sensitivity." +- Avoid making CVaR/OCE sound like promoted selectors. + +### 6. Source shift, multi-distribution, online conformal + +Key local papers: + +- Tibshirani et al. (2019), conformal prediction under covariate shift. +- Barber, Candes, Ramdas, Tibshirani (2023), beyond exchangeability. +- Bhattacharyya and Barber (2026), group-weighted conformal prediction. +- Guan (2023), localized conformal prediction. +- Liu, Levis, Normand, Han (2024), multi-source conformal inference. +- Yang and Jin (2026), multi-distribution robust conformal prediction. +- Gibbs and Candes (2021), adaptive conformal inference. +- Liu et al. (2026), online conformal prediction via universal portfolios. + +What they give CRPTO: + +- A future-work lane and reviewer caveats for group/source/live deployment. + +How to improve paper: + +- Keep A23/A24 as diagnostics. +- Do not promote multi-distribution or online validity without a new protocol. + +## High-Priority Improvement Plan + +### P0: repair precision in cap wording + +Change any exact-sounding text that says the body/default point is under a +`0.345` cap. The exact body/default cap is `0.345083740`; the strict +`<=0.345` row is a neighboring frontier point. + +Recommended wording: + +- "the selected policy sits at the declared approximately 0.345 return-bound + lens, with Markov cap 0.345084" +- "the strict `<=0.345` endpoint earns `$184,800.41`; the body/default balanced + point earns `$184,832.48` with Markov cap `0.345084`" + +Avoid: + +- "highest-return point under cap `<=0.345`" for the body/default row. +- "declared `0.345` Markov cap" unless the next words clarify rounding. + +### P1: add a venue-specific IJDS precedent table + +Add a compact body table or paragraph after the related-work overview: + +| IJDS precedent | Lesson for CRPTO | CRPTO extension | +|---|---|---| +| Das et al. (2023) credit graph ML | IJDS accepts reproducible credit-risk ML. | CRPTO turns credit risk scores into a funded portfolio certificate. | +| Yang and Bi (2025) cost-aware calibration | Calibration matters because downstream costs are asymmetric. | CRPTO prices uncertainty inside a budgeted allocation. | +| Fernandez-Loria and Provost (2022/2025) decision vs estimation | Decision quality is not the same as estimation/prediction quality. | CRPTO evaluates the funded decision, not only PD quality. | +| Falconer et al. (2026) replication-robust analytics | IJDS values robust/reproducible analytics systems. | CRPTO supplies frozen evidence, exact checks, and a reproducibility harness. | + +This helps the editor see fit immediately and helps reviewers place the paper +inside IJDS rather than only OR/ML. + +### P1: make A35 the central result object + +Move the reader quickly from method to A35: + +1. Exact certificate table: what the selected policy achieved. +2. A35 frontier: why this is not a cherry-picked singleton. +3. A19 regret-auditability: why CRPTO is not trying to beat SPO+ on its own + synthetic regret metric. +4. A25 external recipe transfer: why the method is not a Lending Club-only + curiosity. + +The current paper has these pieces; the improvement is ordering and signposting. + +### P1: sharpen abstract to IJDS's three-question template + +Current abstract is good but can be more IJDS-aligned: + +1. Problem/relevance: + "Credit allocation decisions need calibrated probabilities only insofar as + they change funding choices under risk appetite." +2. Methods/results: + "CRPTO maps frozen PD predictions through Mondrian conformal intervals into a + robust LP and finite-grid funded-set audit; on 276,869 OOT loans it earns + `$184.8K` on `$1M` with `V=0.035350`, `Gamma_CP=0.162616`, and Markov cap + `0.345084`." +3. Insight/implication: + "The insight is that uncertainty should be reported as a return-bound + frontier, not as a post-hoc calibration table; reproducible decision + certificates can be audited without retraining a production-style PD model." + +Reduce abstract space devoted to external replications unless needed for +single-dataset defense. + +### P1: make the baseline story a reviewer checklist + +IJDS reviewers will ask whether the paper uses reasonable baselines and +quantifies improvement. CRPTO can answer with a table: + +| Baseline / family | What it optimizes | CRPTO comparison | +|---|---|---| +| Two-stage baseline | Predict then optimize without conformal robust certificate. | CRPTO adds exact funded-set bound and frontier. | +| SPO+ / DFL | Synthetic regret / task-aligned training loss. | SPO+ has lower mean regret; CRPTO has funded-set dollar value and 3/3 verifiable risk controls. | +| P2P profit scoring | Economic loan selection. | CRPTO adds conformal premium and exact alpha-safe audit. | +| P2P robust portfolio | Robust credit allocation. | CRPTO calibrates uncertainty with conformal intervals and exposes finite-grid denominators. | +| Cost-aware calibration | Probability calibration under asymmetric costs. | CRPTO carries calibrated uncertainty into a portfolio decision. | + +The goal is not to claim universal dominance; it is to make the trade-off +impossible to miss. + +### P2: strengthen supplement navigation + +Add a short supplement reader map: + +- Appendix A: proof and Markov boundary. +- Appendix B: robustness/challenger diagnostics. +- Appendix C: active A35--A39 selected-policy closure. +- Appendix D: external recipe transfer. +- Appendix E: reproducibility, DVC, protected stages, and anonymization. + +Then repeat the claim hierarchy in one table: + +| Evidence | Promoted? | Why | +|---|---:|---| +| A35 | Yes | Active finite-grid frontier. | +| A36--A39 | Support | Selected-policy composition/tail/concentration/bootstrap diagnostics. | +| A19 | Support | Regret-auditability contrast. | +| A25--A34 | Support | External economic recipe transfer. | +| A20--A24 | Diagnostics | Tail/source/online objections, not selector changes. | + +### P2: citation hygiene before freeze + +The previous citation audit flagged several body references as partial or +citation-only. This pass reduces risk for `das2023creditgraph`, +`yang2025costaware`, and the Fernandez-Loria/Provost IJDS papers. Still +spot-check before final freeze: + +- `hoeffding1963`, `boucheron2013concentration`, `ghosh2002` +- `goldfarb2003robustportfolio`, `delage2010dro` +- `serrano2016profitscoring`, `zhao2016p2pportfolio` +- any recent credit/IJDS references added after 2026-06-14 + +Do this as source verification, not as a broad literature expansion. + +### P2: prepare response-ready reviewer objections + +Prewrite one paragraph each: + +- "Why not SPO+?" +- "Why not CVaR/OCE as the selector?" +- "Is the result cherry-picked?" +- "What happens under dependence?" +- "Is this a live-production guarantee?" +- "What exactly can be reproduced under double anonymous review?" + +Most answers already exist in the body/supplement; the improvement is to make +them short and reusable. + +## Concrete Editing Checklist + +Before submission: + +1. Replace exact-sounding `0.345` cap wording for the body/default point. +2. Add IJDS precedent paragraph/table with Das, Yang/Bi, Fernandez-Loria/Provost, + and Falconer/Kazempour/Pinson. +3. Move or signpost A35 so the finite-grid frontier appears as the main results + object, not only as a supporting table. +4. Add one compact "what the reviewer should remember" paragraph before the + A19 regret table. +5. Tighten abstract external-replication language to "recipe transfer." +6. Add supplement reader map and promoted/support/diagnostic hierarchy. +7. Check all anonymous review paths: no repo URLs, local paths, author names, + affiliations, or identifying metadata in body/supplement PDFs. +8. Run `just smoke` and `just validate-champion` after any paper edits. +9. Rebuild official submission PDF and check page count/log. + +Do **not** do before submission unless a reviewer or explicit research decision +requires it: + +- Rerun protected champion/search/conformal stages. +- Promote CVaR/OCE, multi-distribution, online, causal, or end-to-end DFL + variants. +- Turn external Prosper/Freddie replications into new exact funded-set + certificates. +- Expand the body with a long literature review. + +## Best Current IJDS Story + +The paper should read like this: + +1. Credit allocation is a decision problem; PD calibration alone is insufficient. +2. Existing IJDS work shows calibration, credit ML, and causal decision framing + matter for data-science decisions. +3. CRPTO contributes the missing bridge: a frozen predictive model becomes a + robust funding decision with a conformal premium and exact finite-grid audit. +4. The selected policy earns `$184,832.48` on `$1M`, with `V=0.035350`, + `Gamma_CP=0.162616`, and Markov cap `0.345084`. +5. The result is not a singleton: A35 exposes 50,010 semantic policies and a + return-bound frontier. +6. The method does not beat SPO+ at SPO+'s own regret target; instead, it buys + auditability, risk controls, and a funded-set dollar certificate. +7. The supplement shows tail, concentration, bootstrap, source, online, and + external-recipe checks without changing the body claim. +8. The reproducibility package is part of the scientific contribution, not just + administration. + +That is the IJDS version of CRPTO: **a reproducible decision certificate for +credit portfolio allocation under conformal uncertainty**. diff --git a/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md b/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md new file mode 100644 index 0000000..2f62787 --- /dev/null +++ b/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md @@ -0,0 +1,100 @@ +# IJDS Exact-Alpha and Calibration-Selection Closeout + +Date: 2026-07-09 + +## Why the active claim changed + +The previous paper-facing alpha sweep did not recompute conformal quantiles at +each alpha. It scaled stored 90% row radii using average-width ratios from a +different conformal family. The resulting alpha-0.01 endpoints were useful as +an exploratory approximation, but they did not support the manuscript's +"exact alpha" language. The former `8/8`, `0.345084` threshold, and 50,010-grid +selection claims were therefore retired rather than cosmetically relabeled. + +## Exact replay + +`src/models/conformal_alpha_grid.py` reconstructs the frozen score-decile +Mondrian recipe from its result payload and recomputes quantiles for each alpha. +At the reference 90% level, replay matches the stored point, low, and high +vectors to at most `6.67e-16`. The exact alpha sweep also exposes why +`alpha = 0.01` is not decision-useful here: average interval width is `0.9882` +and `93.54%` of upper endpoints equal one. + +## Final policy protocol + +The final run is +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. +It uses the exactly replayed 90% interval recipe and a nine-policy round-number +grid: + +- `tau in {0.15, 0.17, 0.19}`; +- `gamma in {0.25, 0.50, 0.75}`; +- linear `q = p + gamma(u-p)` only; +- no cap, tail rule, or uncertainty-aversion penalty; +- point-PD expected net return in the objective; +- conformal `q` only in the portfolio-risk constraint. + +The loader keeps outcomes physically separate from the 12-column policy frame, +which has no default, realized-return, miscoverage, or assumption-conditional +quantity. It requires +at least 99.9% budget use, feasibility of the effective-PD cap, and +`B_u <= 0.28` on November 2017. Five candidates are eligible. Maximizing +expected point-PD objective selects `tau = 0.17, gamma = 0.50`, the +interpretable midpoint `q = (p+u)/2`. The selected row is invariant for caps in +`[0.259036, 0.290491)`. + +An outcome-free December replay selects the same policy. Outcomes are opened +only afterward: weighted default is `0.145650`, weighted miscoverage is +`0.124925`, and endpoint budget is `0.262082`. This independent audit is not a +coverage theorem; it is direct evidence that policy stability does not imply +selected-set conformal validity. + +## What was learned from the challengers + +- The exploratory 25-policy selector chose `gamma = 0.35` and earned + `$181,758.69` OOT, but its threshold was `0.646094` and its policy grid and + cap were harder to defend. +- The final 50/50 policy earns `$179,327.59`, only `$2,431.11` less, while + lowering weighted default from `0.04855` to `0.039375` and the threshold from + `0.646094` to `0.574279`. +- A 75% blend is safer (`0.035875` default; `0.516624` threshold) but earns + `$6,388.08` less than the selected midpoint. +- The matched point-PD policy earns more on the full OOT panel, but its default + rate and endpoint audit are substantially worse. It also beats CRPTO in some + temporal slices, so no universal dominance claim is supportable. + +## Endpoint-screen v7 promotion + +The v6 selector used `B_u + sqrt(0.10) <= 0.60`. Because alpha was fixed, that +screen was algebraically an endpoint cap but was described with the same +Markov expression whose interpretation requires weighted funded-set validity. +The v7 challenger removes that coupling: + +- selection uses the round deterministic cap `B_u<=0.28`; +- November selects and December replays the same nine-row grid; +- the winner is unchanged for caps in `[0.259036, 0.290491)`; +- the loader separates outcomes from a 12-column selector frame with no + statistical-bound columns; +- the OOT loader aligns candidates directly to exact-alpha rows by unique ID, + removing a private script import and redundant intermediate alignment; +- v6 and v7 produce identical 845 funded comparator rows, IDs, allocations, + exposures, interval values, and full/temporal metrics (`max abs diff = 0`); +- A39 now treats 31 origination months as the primary resampling units and + retains funded-loan resampling as a sensitivity. + +No protected DVC stage was rerun because the challenger changes only policy +selection semantics, panel alignment, and downstream evidence. The exact-alpha +artifact and every manifest-protected upstream byte remain the comparison +baseline. + +## Interpretation + +The scientific upgrade is not a larger search. It is a smaller and auditable +decision rule, an exact conformal replay at the level actually used, separation +of point-PD economics from conformal feasibility, and a policy selector whose +inputs can be inspected for outcome leakage. This is the active IJDS narrative. + +The historical OOT panel was inspected during earlier project development. +Accordingly, v7 is described as a retrospective lockbox replay with an +outcome-free, assumption-free final selector conditional on the frozen +conformal recipe, not as a pristine prospective holdout or preregistered trial. diff --git a/docs/research/ijds_literature_expansion_scan_2026-07-08.md b/docs/research/ijds_literature_expansion_scan_2026-07-08.md new file mode 100644 index 0000000..bf11190 --- /dev/null +++ b/docs/research/ijds_literature_expansion_scan_2026-07-08.md @@ -0,0 +1,514 @@ +# IJDS literature expansion scan, 2026-07-08 + +## Scope + +This note records an external and local literature scan for the IJDS version of +CRPTO. It is designed as an editorial input, not as a request to reopen the +frozen champion. The active certificate remains the pool93 finite-grid decision +certificate documented in `docs/research/active_claims_2026-07-04.md`. + +Operational guardrails: + +- Do not modify `EXTRACTION_MANIFEST.json` or artifacts listed there. +- Do not rerun protected champion stages without explicit permission. +- Keep the IJDS body focused on the implemented certificate: finite alpha grid, + robust funding decision, exact violation audit, and economic return. +- Use the supplement for adjacent method families, diagnostics, and future work. + +Sources reviewed: + +- Local corpus: `Papers_tesis` inventory and benchmark snapshot + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/snapshot.md`. +- Current bibliography: `book/references.bib`. +- Current manuscript anchors: `book/chapters/CRPTO_*.qmd`, `paper/submission`, + and the active claims register. +- External web scan on conformal decision-making, contextual optimization, + credit scoring uncertainty, IJDS-adjacent work, and 2025-2026 emerging papers. + +## Bottom line + +The paper already cites the core CRPTO neighborhood well: conformal risk control, +predict-then-calibrate, conformal contextual robust optimization, conformal +robustness control, conformal robust optimization/satisficing, end-to-end +conformal calibration, and credit/P2P decision papers. The most useful additions +are not dozens of new citations. The highest-value improvement is a small set of +strategic anchors that sharpen the IJDS story: + +1. Add one broad contextual optimization survey to show that CRPTO sits inside + the modern prediction-to-decision literature. +2. Add one credit-specific profit/uncertainty paper to show that economic credit + scoring is active, but CRPTO advances from score-level economics to a funded + portfolio decision certificate. +3. Add one non-exchangeability/source-shift conformal reference in the + supplement. +4. Add one post-selection or human-decision conformal limitation reference to + make the paper look honest and current. +5. Keep 2026 inverse/decision-calibrated and decision-aware conformal-set papers + as future-work comparators, not body-level foundations, unless reviewers ask. + +## Recommended additions + +### 1. Sadana et al. 2025, contextual optimization survey + +Candidate: + +- Rahul Sadana, Andrea Delage, Alexandre Forel, Emma Frejinger, Thibaut Vidal. + "A survey of contextual optimization methods for decision-making under + uncertainty." European Journal of Operational Research, 320(2), 2025. +- Sources: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221724002200), + [arXiv](https://arxiv.org/abs/2306.10374). + +Why it matters: + +- It is an OR-facing survey of contextual optimization, prescriptive analytics, + predict-then-optimize, decision-focused learning, and smart predict/estimate- + then-optimize methods. +- It gives IJDS reviewers a clean map for why CRPTO is a decision paper rather + than a classifier leaderboard. + +Recommended placement: + +- Body, related work or positioning paragraph. +- One sentence is enough: CRPTO belongs to contextual/predictive-prescriptive + optimization, but differs by producing an auditable finite-grid robust funding + certificate rather than learning a new end-to-end policy. + +Priority: High. + +### 2. Xu, Kou, and Ergu 2025, profit-based uncertainty in credit scoring + +Candidate: + +- Zhuozhuo Xu, Gang Kou, Daji Ergu. "Profit-based uncertainty estimation with + application to credit scoring." European Journal of Operational Research, + 325(2), 2025. +- Sources: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221725002048), + [IDEAS/RePEc](https://ideas.repec.org/a/eee/ejores/v325y2025i2p303-316.html). + +Why it matters: + +- It is close to our application domain and explicitly links credit scoring, + uncertainty, rejection, and profitability. +- It supports the narrative that predictive uncertainty in lending should be + evaluated economically, while CRPTO moves the unit of decision from + application-level classification/rejection to portfolio funding under a + distribution-free certificate. + +Recommended placement: + +- Body if there is space in the credit/P2P paragraph. +- Otherwise supplement literature table. + +Priority: High. + +### 3. Xu et al. 2024, profit- and risk-driven credit scoring + +Candidate: + +- Zhuozhuo Xu, Yishun Dou, Gang Kou, Daji Ergu. "Profit- and risk-driven credit + scoring under parameter uncertainty: A multiobjective approach." Omega, 125, + 2024. +- Source: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0305048323001688). + +Why it matters: + +- It gives a credit-scoring reference for profit/risk tradeoffs under uncertain + cost-benefit parameters. +- It is useful if we want a short supplement note distinguishing CRPTO from + multiobjective credit-score design: CRPTO certifies a portfolio decision over + existing predictions and realized returns rather than optimizing an application + classifier under assumed parameter uncertainty. + +Recommended placement: + +- Supplement, not body, unless reviewers ask for more credit-scoring economics. + +Priority: Medium. + +### 4. Farinhas et al. 2024, non-exchangeable conformal risk control + +Candidate: + +- Antonio Farinhas, Alessandro Zecchin, Andre Martins, David Grangier. + "Non-Exchangeable Conformal Risk Control." ICLR 2024. +- Source: [arXiv](https://arxiv.org/abs/2310.01262). +- Local PDF already exists in `Papers_tesis/supplement`. + +Why it matters: + +- It directly addresses a limitation readers may raise: calibration/test + exchangeability, time dependence, and distribution shift. +- It can strengthen the supplement language around source-shift diagnostics + without making a new guarantee in the IJDS body. + +Recommended placement: + +- Supplement A23/A24 source-shift and calibration-drift discussion. +- Do not cite it as implemented protection unless we implement and validate the + method. + +Priority: High for supplement, low for body. + +### 5. Hegazy et al. 2025, valid selection among conformal sets + +Candidate: + +- Mahmoud Hegazy, Emma Frejinger, Pierre Pinson, Alexandre Forel. + "Valid Selection among Conformal Sets." 2025. +- Source: [arXiv](https://arxiv.org/abs/2506.20173). + +Why it matters: + +- It is conceptually important for CRPTO because our grid search selects among + candidate decisions and alpha values after evaluating constraints. +- The current paper is careful because it reports finite-grid denominators, + exact violation audit, and does not claim a new post-selection conformal + theorem. This paper helps articulate that boundary. + +Recommended placement: + +- Supplement limitations/future protocol. +- Possible sentence: future CRPTO variants could study formal post-selection + validity for choosing among multiple calibrated sets, whereas the present + paper reports a finite-grid certificate and exact audit for the selected + champion. + +Priority: High for limitations. + +### 6. Hullman et al. 2025, conformal prediction and human decision making + +Candidate: + +- Jessica Hullman, Christopher W. Zamecnik, Yuval Rabin, Fred Hohman. + "Conformal Prediction and Human Decision Making." 2025. +- Source: [arXiv](https://arxiv.org/abs/2503.11709). + +Why it matters: + +- It argues that valid conformal sets are not automatically useful for decisions + unless the downstream objective is explicit. +- This is useful for IJDS framing: CRPTO is not "uncertainty reporting"; it + operationalizes uncertainty through a robust funding decision and economic + audit. + +Recommended placement: + +- Optional short citation in introduction or limitations. +- Use sparingly; avoid turning the paper into a human-factors discussion. + +Priority: Medium. + +### 7. Djeundje, Crook, and Andreeva 2025, dynamic loan portfolio profitability + +Candidate: + +- Viani B. Djeundje, Jonathan Crook, Galina Andreeva. "The devil in the details: + Dynamic prediction of loan portfolio profitability with macroeconomic drivers + through multi-state modelling." European Journal of Operational Research, + 327(2), 2025. +- Sources: [IDEAS/RePEc](https://ideas.repec.org/a/eee/ejores/v327y2025i2p703-715.html), + [University of Edinburgh](https://www.research.ed.ac.uk/en/publications/the-devil-in-the-details-dynamic-prediction-of-loan-portfolio-pro/). + +Why it matters: + +- It is a recent credit-portfolio profitability paper rather than a pure + application-level classifier paper. +- It can help if a reviewer wants more credit portfolio literature around + dynamic profitability and macroeconomic drivers. + +Recommended placement: + +- Supplement only, unless the introduction is rewritten to emphasize + macro-sensitive portfolio profitability. +- Do not use it to imply that CRPTO currently models macro transitions; it does + not. + +Priority: Medium-low. + +## Emerging close comparators to monitor + +These papers are close to CRPTO but are very recent, mostly preprint-era, or in +different domains. They should strengthen future-work positioning rather than +drive the IJDS body. + +### Zhou and Zhu 2026, inverse conformal risk control for decision robustness + +- "Calibrating Decision Robustness via Inverse Conformal Risk Control." +- Sources: [arXiv](https://arxiv.org/abs/2510.07750), + [OpenReview](https://openreview.net/forum?id=lV4tqcVIyx&referrer=%5Bthe+profile+of+Shixiang+Zhu%5D%28%2Fprofile%3Fid%3D~Shixiang_Zhu1%29). +- Value: Very close to CRPTO because it treats the robustness level itself as + the calibrated object and reports finite-sample guarantees on miscoverage and + regret for robust predict-then-optimize policy families. +- Recommendation: strongest 2026 future-work comparator. It is too new to make + it a foundation of the current body, but it is the cleanest citation if we add + one sentence about future calibration of robustness levels. + +### Stratigakos et al. 2026, decision-calibrated prediction sets + +- "Decision-calibrated prediction sets for robust power system operations." +- Source: [arXiv](https://arxiv.org/abs/2606.02081). +- Value: Closest phrase-level comparator for calibrating prediction sets by + downstream decision reliability. +- Recommendation: future-work comparator only. + +### Chen, Zhou, and Zhu 2026, learning polyhedral conformal sets for RO + +- "Learning Polyhedral Conformal Sets for Robust Optimization." +- Source: [arXiv](https://arxiv.org/abs/2605.08506). +- Value: Decision-aware conformal uncertainty sets for robust optimization. +- Recommendation: cite in future work if adding a paragraph on learned + uncertainty-set geometry. + +### Wang and Dobriban 2026, optimal decisions from prediction sets + +- "Optimal Decision-Making Based on Prediction Sets." +- Source: [arXiv](https://arxiv.org/abs/2602.00989). +- Value: Decision-theoretic framework for downstream use of prediction sets. +- Recommendation: monitor; useful if reviewers ask for more theory around + prediction sets as decision objects. + +### Huang, Farzaneh, and Simeone 2026, OCE risk-controlling prediction sets + +- "Optimized Certainty Equivalent Risk-Controlling Prediction Sets." +- Source: [arXiv](https://arxiv.org/abs/2602.13660). +- Value: OCE/CVaR-style risk-control extension. +- Recommendation: supplement/future work near the existing OCE/CVaR diagnostics. + +### Baesens et al. 2026, foundation models for credit risk prediction + +- "Foundation Models for Credit Risk Prediction: A Game Changer?" +- Source: [arXiv](https://arxiv.org/abs/2605.18147). +- Value: Useful for monitoring PD-layer baselines in credit risk. +- Recommendation: do not add to IJDS body now. CRPTO does not claim a PD model + leaderboard. + +## Already-covered nearest neighbors + +The current corpus and official submission already include the most important +methodological neighbors. These should remain the core comparison set: + +- Patel et al. 2024, conformal contextual robust optimization. +- Sun et al. 2024, predict-then-calibrate. +- Hu et al. 2026, conformal robustness control. +- Zhao et al. 2025/2026, conformal robust optimization and satisficing. +- Yeh et al. 2025/2026, conformal risk training and end-to-end conformal + calibration. +- Bao et al. 2025, CROMS. +- Zhou et al. 2025/2026, CREDO and CREME. +- Yang and Bi 2025, cost-aware calibration. +- Liu et al. 2026, online conformal portfolio selection. +- Yang and Jin 2026, multidistribution conformal prediction. + +The key editorial move is to distinguish CRPTO from each in one sentence: + +- Not a new PD scorer. +- Not a generic conformal-set method. +- Not end-to-end training. +- Not online rebalancing. +- Not a multidistribution fairness theorem. +- A finite-grid robust portfolio decision certificate with realized-return and + exact-violation audit. + +## IJDS-specific venue scan + +Relevant IJDS-adjacent sources are useful mainly for framing, not for core +method positioning: + +- Wiberg et al. 2025, "Synergizing Artificial Intelligence and Operations + Research." IJDS. Source: + [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2025.0077). +- Morucci et al. 2022, "A Robust Approach to Quantifying Uncertainty in Matching + Problems of Causal Inference." IJDS. Source: + [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2022.0020). +- "Rethinking Cost-Sensitive Classification in Deep Learning via Adversarial + Data Augmentation." IJDS. Source: + [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2022.0033). + +Recommendation: + +- Do not overload the paper with IJDS self-citations. +- If a cover letter or response-to-reviewers needs venue fit, Wiberg et al. + 2025 is a concise AI+OR anchor. +- Morucci et al. 2022 can be mentioned only if a reviewer asks about robust + uncertainty quantification precedent in IJDS. +- The cost-sensitive classification IJDS paper is less aligned than Yang and Bi + 2025 for our current framing. + +## Recent credit-risk papers to leave out unless needed + +The 2025-2026 EJOR credit-risk stream is active. Several papers are useful for +background but should not crowd the IJDS body: + +- Distaso, Roccazzella, and Vrins 2025, "Business cycle and realized losses in + the consumer credit industry." Source: + [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221724009688). +- Ballegeer, Bogaert, and Benoit 2025, "Evaluating the stability of model + explanations in instance-dependent cost-sensitive credit scoring." Source: + [ScienceDirect](https://www.sciencedirect.com/science/article/abs/pii/S0377221725004230). +- Baesens et al. 2026, foundation models for credit risk prediction. Source: + [arXiv](https://arxiv.org/html/2605.18147v1). + +Recommendation: + +- Use these only if a reviewer asks for macro-credit loss, explanation + stability, or modern PD-model background. +- They do not change the CRPTO contribution because the paper is not proposing a + new PD learner. + +## Exact manuscript insertion plan + +Keep the body lean. The highest-value body additions are two citations: + +1. Add Sadana et al. 2025 to the contextual optimization / prescriptive + analytics positioning. +2. Add Xu, Kou, and Ergu 2025 to the credit scoring uncertainty / profit + paragraph. + +Then use the supplement for three limitation/future-work citations: + +3. Farinhas et al. 2024 in source-shift/non-exchangeability. +4. Hegazy et al. 2025 in post-selection validity. +5. Zhou and Zhu 2026 as the preferred emerging future-work comparator; if space + allows, add one of Stratigakos et al. 2026, Chen et al. 2026, or Wang and + Dobriban 2026 for decision-calibrated conformal-set geometry/use. + +Suggested body language: + +> CRPTO sits within contextual optimization and prescriptive analytics, where +> predictions and downstream decisions are optimized jointly or sequentially, +> but it differs by auditing a finite grid of robust funding decisions rather +> than training a new decision rule. + +> Recent credit-scoring work evaluates uncertainty through profit and rejection +> decisions; CRPTO instead evaluates uncertainty at the funded-portfolio level, +> where the reported object is an economically realized decision certificate. + +Suggested supplement language: + +> The present certificate assumes the calibration/audit design documented in +> the finite-grid protocol. Extensions to non-exchangeable conformal risk control +> and formal selection among multiple conformal sets are natural next steps, but +> are not claimed by the current champion. + +## Bibliography action list + +Add or verify BibTeX entries for: + +- `sadana2025contextual` +- `xu2025profit_uncertainty_credit` +- `xu2024profit_risk_credit` +- `farinhas2024nonexchangeable_crc` +- `hegazy2025valid_selection_conformal_sets` +- `hullman2025conformal_human_decision` +- `djeundje2025dynamic_loan_portfolio_profitability` +- `zhou2026inverse_crc_decision_robustness` +- `stratigakos2026decision_calibrated_sets` +- `chen2026polyhedral_conformal_ro` +- `wang2026optimal_decision_prediction_sets` +- `huang2026oce_rcps` +- `baesens2026foundation_credit_risk` + +Before adding all of them, apply a manuscript budget rule: + +- Body: at most two new citations unless a paragraph is rewritten. +- Supplement: three to six citations are acceptable. +- Future-work table: emerging 2026 papers are acceptable if clearly labeled as + future work and not as implemented guarantees. + +## Citation synchronization note + +The official submission `.tex` currently contains several compact citation +anchors that are not all mirrored in the source `.qmd` files. Examples observed +during this scan include: + +- `angelopoulos2024foundations` +- `bates2021rcps` +- `zhou2024` +- `sun2024ptc` +- `boosting2025default` +- `yeh2025training` + +If the paper is regenerated from Quarto, these anchors could be lost unless the +QMD sources are synchronized. The safest path is to update QMD first, regenerate +official submission artifacts, and then re-run the IJDS compile checks. + +## Do-not-add list for the current IJDS submission + +Do not add broad or weakly related references unless a reviewer asks: + +- Generic LLM/tabular-model credit-risk papers that do not affect the CRPTO + certificate. +- Generic conformal prediction surveys beyond the already cited foundations. +- Extra IJDS venue papers solely for journal signaling. +- Additional classifier benchmarking papers unless they directly affect the + LendingClub decision framing. + +## Decision recommendation + +For the current IJDS revision, implement the following compact literature +upgrade: + +1. Add Sadana et al. 2025 and Xu, Kou, and Ergu 2025 to the body. +2. Add Farinhas et al. 2024 and Hegazy et al. 2025 to the supplement. +3. Add one 2026 emerging decision-calibrated conformal reference to future work. +4. Keep all language explicit that these are positioning and future-work + references; they do not change the frozen champion or its claims. + +This gives reviewers the right signals: the paper is current, aware of adjacent +decision-calibrated conformal work, and still disciplined about what it actually +implements and certifies. + +## Post-read implementation note, 2026-07-08 + +After the missing PDFs were downloaded manually to `Downloads`, they were copied +into `Papers_tesis/supplement` with normalized names: + +- `Xu Kou Ergu 2025 - Profit-based uncertainty estimation with application to credit scoring.pdf` +- `Xu et al 2024 - Profit- and risk-driven credit scoring under parameter uncertainty.pdf` +- `Djeundje Crook Andreeva 2025 - Dynamic prediction of loan portfolio profitability.pdf` +- `Wiberg Dai Lam Kulkarni 2025 - Synergizing AI and OR.pdf` + +The expanded `academic-pdf-intake` inventory now sees 100 PDFs in scope: +97 under `Papers_tesis` and the three active CRPTO PDFs. The post-read +manuscript update keeps the body claim unchanged and implements only narrative +and boundary changes: + +- Body: CRPTO is framed as a contextual-optimization credit instance, anchored + by credit-scoring uncertainty/profit literature and the AI/OR IJDS perspective. +- Theory: post-selection conformal validity is named explicitly as future + protocol, not as an implicit property of the finite-grid frontier. +- Supplement: non-exchangeable CRC, valid selected conformal sets, + decision-calibrated/inverse conformal robustness, learned polyhedral sets, + OCE-RCPS, and recent credit-profitability/explanation-stability papers are + mapped to diagnostics or future-work boundaries. + +No frozen champion artifact, manifest entry, or protected DVC stage is changed +by this literature update. + +## Venue recheck, 2026-07-09 + +The current [IJDS submission guidelines](https://pubsonline.informs.org/page/ijds/submission-guidelines) +still require the Data + Models + Decisions + Implications synthesis and now +state an explicit abstract sequence: problem/data-science relevance, method and +results, then learned insight and implication. The CRPTO abstract already +covered the first two elements but ended by repeating pipeline architecture. +Its closing sentences now state the learned decision insight and committee use: +the return-bound frontier makes predictive uncertainty actionable, while the +certificate separates exact funded-set accounting from its weighted-validity +assumption. + +Two recent accepted IJDS papers provide a useful style check rather than new +citation requirements: + +- [Robust and Interpretable Policy Learning for Manufacturing Process Parameters](https://pubsonline.informs.org/doi/10.1287/ijds.2024.0041) + leads from a concrete decision problem to a named policy method, robustness, + interpretability and practical deployment evidence. +- [Using Operational Data Analytics for Planning Decisions Under Uncertainty](https://pubsonline.informs.org/doi/10.1287/ijds.2024.0051) + makes the estimate-then-optimize gap explicit, compares against several + decision baselines and closes with real-data effectiveness. + +CRPTO already follows those structural signals through the funded-set decision, +the exact certificate, the A19 regret comparator and the managerial implication. +No additional venue self-citation is warranted: the existing Das et al. IJDS +credit-risk anchor and Wiberg et al. AI/OR anchor are sufficient, and adding +more would crowd the body without changing the novelty boundary. diff --git a/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md b/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md index 7c7bf52..209815d 100644 --- a/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md +++ b/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md @@ -1,5 +1,10 @@ # IJDS simplification and cleanup audit - 2026-07-06 +> Certificate metrics in this historical cleanup memo were superseded on +> 2026-07-09 by `pool93_certificate_semantics_v2_2026-07-09.md`. The cleanup +> decisions remain valid; the active paper uses the policy-aware A35 frontier +> and matched point-PD baseline A40. + Scope: body manuscript, IJDS submission `.tex`, online supplement, code refactor posture, and local repository weight. This memo follows the 2026-07-05 full audit but focuses on reader-facing parsimony: remove technical @@ -89,8 +94,8 @@ Recommended new-run shape if Carlos chooses that route: 2. Write outputs under a new experiment path; do not overwrite the current frozen A35--A39 or model/conformal artifacts. 3. Compare against the current selected policy on: return, `V(0.01)`, - `Gamma_CP`, Markov cap, exact violation, table/page simplicity, and code path - length. + `Gamma_CP`, `Gamma_res`, exact loss threshold, realized risk-tolerance + excess, table/page simplicity, and code-path length. 4. Promote only if the paper becomes materially simpler or the result is materially easier to defend. A few basis points of metric loss may be fine, but only if the new claim is easier to explain. diff --git a/docs/research/literature_reference_audit_2026-06-14.md b/docs/research/literature_reference_audit_2026-06-14.md index a0822c3..25c4fc3 100644 --- a/docs/research/literature_reference_audit_2026-06-14.md +++ b/docs/research/literature_reference_audit_2026-06-14.md @@ -75,3 +75,24 @@ The weaker area is not the main claim; it is citation hygiene. Several classical or recent references are being used as positioning anchors without a local note proving close reading. They can stay for now, but they deserve a targeted pre-freeze source check rather than another broad literature expansion. + +## 2026-07-07 targeted pre-freeze spot-check + +The pre-freeze spot-check resolved the highest-risk citation-only items that +support the IJDS body and supplement theory/positioning language: + +| Key | Source checked | Result | +|---|---|---| +| `ghosh2002` | Taylor & Francis DOI landing page, `10.1198/000313002119`. | Metadata matches BibTeX; appropriate for Markov-related probability-inequality language. | +| `hoeffding1963` | Taylor & Francis DOI landing page, `10.1080/01621459.1963.10500830`. | Metadata matches BibTeX; appropriate for bounded-sum tightening language when independence is stated. | +| `boucheron2013concentration` | Oxford Academic book page / DOI `10.1093/acprof:oso/9780199535255.001.0001`. | Metadata matches BibTeX; appropriate as a modern concentration reference. | +| `goldfarb2003robustportfolio` | INFORMS PubsOnLine DOI landing page, `10.1287/moor.28.1.1.14260`. | Metadata matches BibTeX; appropriate for robust portfolio selection under parameter uncertainty. | +| `delage2010dro` | INFORMS PubsOnLine DOI landing page, `10.1287/opre.1090.0741`. | Metadata matches BibTeX; appropriate for DRO/moment-uncertainty positioning. | +| `zhao2016p2pportfolio` | ACM/KDD DOI landing page, `10.1145/2939672.2939861`. | Title updated in BibTeX to include the subtitle "A Multi-Objective Perspective"; appropriate for P2P portfolio-selection positioning. | +| `serrano2016profitscoring` | ScienceDirect DOI landing page, `10.1016/j.dss.2016.06.014`. | Author accents corrected in BibTeX; appropriate for P2P profit-scoring positioning. | +| `falconer2026replication` | INFORMS/IJDS DOI landing page, `10.1287/ijds.2025.0075`. | Metadata matches BibTeX; appropriate for the replication-robust IJDS precedent. | + +No broad literature expansion is recommended before submission. Remaining +recent-credit references (`das2023creditgraph`, `yang2025costaware`, +`boosting2025default`, `zheng2026twostage`) are lower-risk positioning anchors; +verify only if their exact body sentences become more specific. diff --git a/docs/research/pool93_certificate_semantics_v2_2026-07-09.md b/docs/research/pool93_certificate_semantics_v2_2026-07-09.md new file mode 100644 index 0000000..02f71f2 --- /dev/null +++ b/docs/research/pool93_certificate_semantics_v2_2026-07-09.md @@ -0,0 +1,132 @@ +# Auditoria consolidada del certificado pool93 v2 - 2026-07-09 + +## Decision + +Se promueve la semantica `certificate-semantics-v2` como fuente activa para +A35, la gobernanza pool93 y el baseline A40. No cambia el modelo PD, calibrador, +intervalos conformales, asignacion seleccionada, retorno del body, grilla alpha +ni denominadores. Corrige la lectura del endpoint para policies no lineales y +reemplaza el comparador Lending Club mal rotulado por una baseline point-PD +emparejada. + +Tag activo: + +`champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` + +## Hallazgo 1: descomposicion policy-aware + +La formula historica + +`B_u = tau + (1 - gamma) * Gamma_CP` + +es exacta solo para un blend lineal cuyo cap efectivo liga. La frontera tambien +contiene policies `capped`, `tail` y `segment_relative_tail`; en ellas `gamma` +no basta para reconstruir el endpoint. + +Para cualquier score efectivo declarado `q_i` con +`p_hat_i <= q_i <= u_i`, la identidad general es: + +```text +Gamma_CP = sum_i w_i (u_i - p_hat_i) +Gamma_int = sum_i w_i (q_i - p_hat_i) +Gamma_res = sum_i w_i (u_i - q_i) +Gamma_CP = Gamma_int + Gamma_res +B_u = sum_i w_i q_i + Gamma_res +B_u <= tau + solver_slack + Gamma_res +T_Markov = B_u + sqrt(alpha) +``` + +`T_Markov` es un umbral de evento probabilistico bajo weighted funded-set +validity; no es un cap determinista. El campo historico `violation` mide exceso +realizado de default sobre `tau`, no violacion de la identidad +`sum(wY) <= B_u + V`. + +## Punto seleccionado + +En `alpha=0.01`, la asignacion seleccionada conserva: + +| Cantidad | Valor | +|---|---:| +| Retorno realizado | `$184,832.48` | +| Filas financiadas | `314` | +| `V` / default ponderado | `0.035350` | +| PD puntual ponderada | `0.082468` | +| Score efectivo ponderado | `0.171500` | +| `Gamma_CP` | `0.162616` | +| `Gamma_int` | `0.089032` | +| `Gamma_res` | `0.073584` | +| Endpoint exacto `B_u` | `0.245084` | +| Umbral Markov exacto | `0.345084` | +| Exceso realizado sobre `tau` | `0.000000` | +| Pass grilla alpha | `8/8` | + +El cap row-level de la policy `capped_blended_uncertainty` esta inactivo en las +314 filas financiadas. Por eso la formula lineal coincide numericamente en este +punto, aunque no sea valida como formula universal de la frontera. + +## Auditoria de A35 sin nueva busqueda + +La reconstruccion lee las estadisticas suficientes de seis evaluaciones exactas +ya existentes. No ejecuta HPO, busqueda de policy, generacion conformal ni solve +de portafolio. + +- filas crudas: `51,678`; +- policies semanticas deduplicadas: `50,010`; +- policies elegibles all-alpha y sobre floor: `27,508`; +- seleccion del body: sin cambio; +- thresholds con cambio material: `10,423`; +- policies tail con understatement: `2,866`; +- understatement maximo: `0.241324`; +- antiguas filas `<=0.50` que exceden `0.50` exacto: `716`. + +Los modos afectados son `tail_blended_uncertainty` y +`segment_relative_tail_blended_uncertainty`. El endpoint de maximo retorno sigue +ganando `$223,458.14`, pero su endpoint es `0.597056` y su umbral Markov exacto +es `0.697056`. El body y todos los denominadores permanecen iguales. + +## Hallazgo 2: baseline point-PD A40 + +El campo historico `price_of_robustness=-10.56%` comparaba contra un solve +rotulado `nonrobust` que aun heredaba una restriccion `pd_high`. Una +recomputacion preliminar a `tau=0.175` sirvio para detectar el problema, pero no +era el contraste final del body y se retira como superficie activa. + +A40 resuelve el contraste correcto con: + +- los mismos `276,869` candidatos; +- presupuesto `$1M`; +- misma concentracion, `tau=0.1715`, LGD, solver y controles operativos; +- point PD en objetivo y restriccion como unica diferencia semantica; +- outcomes OOT usados solo despues del solve. + +| Policy | Retorno | Funded | Default / `V` | `Gamma_CP` | `B_u` | Threshold | +|---|---:|---:|---:|---:|---:|---:| +| Point-PD two-stage LP | `$196,369.14` | `225` | `0.118400` | `0.526736` | `0.680579` | `0.780579` | +| CRPTO seleccionado | `$184,832.48` | `314` | `0.035350` | `0.162616` | `0.245084` | `0.345084` | + +CRPTO cede `$11,536.66` (`5.875%`) y reduce default/miscoverage en `0.08305` +(8.305 puntos porcentuales) y el threshold en `0.435495` (43.55 puntos). Ambos +funded sets quedan debajo de `tau`; solo CRPTO pasa el screen tight +`V <= sqrt(0.01)`. + +## Limite del claim + +A40 es una auditoria emparejada sobre un OOT historico congelado. No demuestra +causalidad, significancia prospectiva ni dominancia universal. A35 es una +frontera de grilla finita, no un optimo continuo. La probabilidad del teorema +requiere weighted funded-set validity; el draw observado audita `V` pero no +prueba por si solo ese supuesto. + +## Artefactos promovidos + +- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` +- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex` +- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` +- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex` +- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` +- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` + +Las filas financiadas de A40 viven bajo `data/processed/experiments/`, no en el +directorio de modelos. Los artefactos antiguos se conservan como provenance, +pero no son fuentes de claims activos. diff --git a/docs/research/pool93_tail_risk_closeout_2026-07-02.md b/docs/research/pool93_tail_risk_closeout_2026-07-02.md index 7b7ab88..b081f6c 100644 --- a/docs/research/pool93_tail_risk_closeout_2026-07-02.md +++ b/docs/research/pool93_tail_risk_closeout_2026-07-02.md @@ -2,6 +2,10 @@ Date: 2026-07-02 +Certificate terminology was synchronized on 2026-07-09 with the policy-aware +A35 audit. A37--A39 values remain valid for the fixed selected allocation; +`0.345084` is now called the exact loss threshold, not a generic cap. + This memo closes the post-promotion caveat that tail-risk, cluster-bound, and bootstrap diagnostics should not be cited as pool93-specific unless regenerated from the selected pool93 funded allocation. @@ -28,14 +32,15 @@ At baseline `LGD = 0.45`, the selected body allocation has: - weighted default rate / `V`: `0.035350` - realized CVaR95 loss rate: `0.276211` - decision-time CVaR95 loss rate: `0.218140` -- Markov cap: `0.345084` +- exact loss threshold at `alpha = 0.01`: `0.345084` Across the LGD grid, repriced return ranges from `$188,367.48` at `LGD = 0.35` to `$179,529.98` at `LGD = 0.60`. ## A38 Cluster-Bound Repricing -At `alpha = 0.01` and `delta = 0.10`, Markov's body threshold is `0.100000`. +At `alpha = 0.01` and `delta = 0.10`, the distribution-free Markov increment +is `sqrt(alpha) = 0.100000`. The regenerated cluster-aware Hoeffding thresholds are: - period: `0.395502` @@ -43,9 +48,10 @@ The regenerated cluster-aware Hoeffding thresholds are: - period-grade: `0.281247` - score-vintage: `0.348546` -None is tighter than Markov. This supports the manuscript's theory boundary: -Markov remains the body-level distribution-free statement, while cluster-aware -tightening is shown as an assumption-priced sensitivity. +None is tighter than the distribution-free Markov step. This supports the +manuscript's theory boundary: the policy-aware certificate remains the +body-level distribution-free statement, while cluster-aware tightening is +shown as an assumption-priced sensitivity. ## A39 Fixed-Allocation Bootstrap @@ -73,6 +79,6 @@ search. A37--A39 are selected-allocation risk-profile audits. They do not change the pool93 body selector, do not make CVaR/OCE the optimized objective, and do not turn bootstrap intervals into a conformal guarantee. The paper-facing claim -remains the finite-grid return-bound certificate in A35 plus the exact funded-set -audit; A37--A39 close reviewer questions about the selected point's tail, +remains the finite-grid policy-aware decision certificate in A35 plus the exact +funded-set audit; A37--A39 close reviewer questions about the selected point's tail, concentration, and empirical contribution profile. diff --git a/justfile b/justfile index 336dce2..87b41d8 100644 --- a/justfile +++ b/justfile @@ -32,9 +32,30 @@ fmt: type-check: uv run mypy src scripts +# Fast type checker from Astral. Daily active-scope use remains advisory while +# ty matures; the clean full scope is blocking in the final submission gate. +type-advisory: + @uv run python scripts/run_ty_advisory.py --scope active + +type-advisory-full: + @uv run python scripts/run_ty_advisory.py --scope full --fail-on-diagnostics --output reports/ci/ty-advisory-full.txt + +complexity-report: + uvx radon cc src scripts -s -n D + +api-docs-core: + uv run --with pdoc pdoc src.optimization.portfolio_model src.optimization.policy_evaluation src.optimization.policy_selection src.models.conformal_alpha_grid src.models.calibration src.evaluation.backtesting src.evaluation.fairness --docformat google --output-directory reports/api-docs --no-browser + +hooks-check: + uv run pre-commit validate-config + uvx prek validate-config .pre-commit-config.yaml + # Fast smoke: paper-final guardrails + Quarto book guardrails smoke: - uv run pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py -q + uv run pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py tests/test_publication_integrity.py -q + +publication-integrity: + uv run python scripts/check_publication_integrity.py test: uv run pytest -q @@ -56,7 +77,19 @@ evidence: journal-package: uv run python scripts/build_crpto_journal_package.py -paper-export: tables figures evidence journal-package book +ijds-evidence: + uv run python scripts/build_ijds_calibration_selected_evidence.py + +# Explicit methodology replays. These write only to versioned experiment paths. +ijds-exact-alpha: + uv run python scripts/experiments/run_ijds_exact_alpha_grid_challenger.py --config configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml + +ijds-policy-challenger: + uv run python scripts/experiments/run_ijds_calibration_selected_policy_challenger.py --config configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml + +ijds-active-replay: ijds-exact-alpha ijds-policy-challenger ijds-evidence + +paper-export: tables figures evidence journal-package ijds-evidence book # IJDS-oriented manuscript body (HTML writing preview). paper-ijds: @@ -69,6 +102,13 @@ paper-ijds-supplement: # Render the current submission-shaped manuscript surfaces. paper-submission: paper-ijds paper-ijds-supplement +# Compile and scan the official INFORMS/IJDS LaTeX handoff draft. +paper-submission-official: + @uv run python scripts/compile_ijds_submission.py + +# Final local IJDS gate before freezing or uploading. +submission-check: ijds-evidence publication-integrity lint type-check type-advisory-full test validate-champion paper-submission paper-submission-official + # IJDS-oriented manuscript body (local HTML-print PDF verification draft). paper-ijds-pdf: uv run python scripts/render_submission_pdf_previews.py --body-only diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json new file mode 100644 index 0000000..d5f5bcb --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json @@ -0,0 +1,291 @@ +{ + "allocation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_selected_policy_full_oot_allocations.parquet", + "calibration_audit": { + "claim_boundary": "Independent post-selection decision audit; not a selected-set coverage theorem.", + "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_policy_holdout_audit.csv", + "grid_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_policy_audit_grid.parquet", + "outcome_free_selected_candidate_id": "linear-005", + "period": "2017-12", + "policy_evaluations": [ + { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "endpoint_budget": 0.2620820323280323, + "expected_objective": 103567.0099590662, + "gamma": 0.5, + "gamma_cp": 0.1841640646560647, + "gamma_internalized": 0.09208203232803233, + "gamma_residual": 0.09208203232803232, + "markov_loss_threshold": 0.5783097983448702, + "min_budget_utilization": 0.0, + "n_funded": 193, + "n_panel": 20695, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "period": "2017-12", + "policy_mode": "blended_uncertainty", + "realized_return": 53313.04791145165, + "realized_risk_tolerance_excess": 0.0, + "risk_tolerance": 0.17, + "role": "calibration_selected", + "screen_V_leq_sqrt_alpha": true, + "screen_risk_excess_leq_alpha": true, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 999999.9999999999, + "uncertainty_aversion": 0.0, + "weighted_miscoverage": 0.12492500000000001, + "weighted_outcome": 0.14565, + "weighted_pd_effective": 0.16999999999999998, + "weighted_pd_point": 0.07791796767196764 + }, + { + "candidate_id": "linear-006", + "delta_cap_quantile": 1.0, + "endpoint_budget": 0.20350382300175537, + "expected_objective": 97897.74942687502, + "gamma": 0.75, + "gamma_cp": 0.13401529200702159, + "gamma_internalized": 0.10051146900526618, + "gamma_residual": 0.033503823001755396, + "markov_loss_threshold": 0.5197315890185933, + "min_budget_utilization": 0.0, + "n_funded": 191, + "n_panel": 20695, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "period": "2017-12", + "policy_mode": "blended_uncertainty", + "realized_return": 38379.50087450525, + "realized_risk_tolerance_excess": 0.0, + "risk_tolerance": 0.17, + "role": "incumbent_linear", + "screen_V_leq_sqrt_alpha": true, + "screen_risk_excess_leq_alpha": true, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 1000000.0, + "uncertainty_aversion": 0.0, + "weighted_miscoverage": 0.134525, + "weighted_outcome": 0.15525, + "weighted_pd_effective": 0.17000000000000004, + "weighted_pd_point": 0.06948853099473384 + }, + { + "candidate_id": "point-pd", + "delta_cap_quantile": 1.0, + "endpoint_budget": 0.8880705762941006, + "expected_objective": 141162.2113059011, + "gamma": 0.0, + "gamma_cp": 0.7180705762941006, + "gamma_internalized": 0.0, + "gamma_residual": 0.7180705762941006, + "markov_loss_threshold": 1.2042983423109386, + "min_budget_utilization": 0.0, + "n_funded": 169, + "n_panel": 20695, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "period": "2017-12", + "policy_mode": "point_estimate", + "realized_return": 89732.35130590109, + "realized_risk_tolerance_excess": 0.015650000000000025, + "risk_tolerance": 0.17, + "role": "point_pd_matched_tau", + "screen_V_leq_sqrt_alpha": true, + "screen_risk_excess_leq_alpha": true, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 1000000.0, + "uncertainty_aversion": 0.0, + "weighted_miscoverage": 0.058300000000000005, + "weighted_outcome": 0.18565000000000004, + "weighted_pd_effective": 0.16999999999999996, + "weighted_pd_point": 0.16999999999999996 + } + ], + "same_policy_selected": true + }, + "calibration_metadata": { + "audit_period": "2017-12", + "audit_rows": 20695, + "calibration_fit_rows": 142550, + "calibration_holdout_end": "2017-12-01", + "calibration_holdout_rows": 35638, + "calibration_holdout_start": "2017-11-01", + "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", + "outcomes_isolated_until_post_selection_audit": true, + "partition": "score_decile_mondrian", + "selection_period": "2017-11", + "selection_rows": 14943, + "target_alpha": 0.1, + "used_alpha": 0.095 + }, + "claim_boundary": "The final tagged rule selects among nine round-number policies on November 2017 without exposing the selector to outcomes or assumption-conditional statistics. December 2017 independently replays the outcome-free selector and audits the already-fixed decision. The audit is diagnostic rather than a selected-set coverage theorem. Earlier development inspected the static OOT corpus, so the January 2018--September 2020 evaluation remains a transparent retrospective lockbox replay, not a pristine prospective trial, causal estimate, or live-deployment guarantee.", + "config_path": "configs\\experiments\\champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml", + "config_sha256": "9df5445defe22044b235dcc0f21bc8f5b734d35c73f916a09e1fe60060f84cc0", + "contrasts": { + "2020+": { + "default_delta_vs_incumbent": -0.010000000000000009, + "default_delta_vs_point": 0.06687499999999999, + "return_cost_vs_point": 118939.59578289572, + "return_delta_vs_incumbent": 16526.81992213076, + "selected_markov_threshold": 0.580689837874186, + "selected_realized_return": 99689.53961659266, + "selected_weighted_outcome": 0.08377499999999999, + "threshold_delta_vs_point": -0.5969627812854879 + }, + "full_oot": { + "default_delta_vs_incumbent": 0.003500000000000003, + "default_delta_vs_point": -0.07902500000000001, + "return_cost_vs_point": 17041.554867740255, + "return_delta_vs_incumbent": 6388.080277992645, + "selected_markov_threshold": 0.5742788554403055, + "selected_realized_return": 179327.5851322598, + "selected_weighted_outcome": 0.03937500000000001, + "threshold_delta_vs_point": -0.663266349105067 + } + }, + "design": { + "alpha": 0.1, + "audit_period": "2017-12", + "budget": 1000000.0, + "combine_years_from": 2020, + "endpoint_budget_cap": 0.28, + "lgd": 0.45, + "max_concentration": 0.25, + "period_order": [ + "2018H1", + "2018H2", + "2019H1", + "2019H2", + "2020+" + ], + "selection_min_budget_utilization": 0.999, + "selection_period": "2017-11", + "selection_rule": "maximize expected point-PD objective on November 2017 under a deterministic endpoint-budget cap of 0.28, the effective-PD cap, and full budget use; reserve December 2017 for an outcome-free selector stability replay and a post-selection decision audit" + }, + "endpoint_cap_stability": { + "cap_lower_inclusive": 0.25903604939435104, + "cap_upper_exclusive": 0.29049078888716334, + "declared_endpoint_budget_cap": 0.28, + "margin_to_lower_boundary": 0.020963950605648984, + "margin_to_upper_boundary": 0.010490788887163316, + "selected_candidate_id": "linear-005", + "selected_endpoint_budget": 0.25903604939435104 + }, + "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_selected_policy_oot_evaluation.csv", + "generated_at_utc": "2026-07-10T04:24:39.860761+00:00", + "grid_size": 9, + "incumbent_policy": { + "candidate_id": "linear-006", + "delta_cap_quantile": 1.0, + "gamma": 0.75, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "oot_alignment": { + "alignment_mode": "id", + "candidate_path": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\data\\processed\\test_fe.parquet", + "exact_alpha_grid_path": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1\\conformal\\exact_alpha_grid.parquet" + }, + "recipe": { + "partition": "score_decile_mondrian", + "partition_probability_source": "calibrated", + "reference_target_alpha": 0.1, + "reference_used_alpha": 0.095 + }, + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7", + "schema_version": "2026-07-09.7", + "selected_calibration_metrics": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "effective_pd_cap_slack": 0.0, + "endpoint_budget": 0.25903604939435104, + "expected_objective": 99387.12330098984, + "gamma": 0.5, + "gamma_cp": 0.17807209878870203, + "gamma_internalized": 0.08903604939435102, + "gamma_residual": 0.089036049394351, + "markov_loss_threshold": 0.575263815411189, + "min_budget_utilization": 0.0, + "n_funded": 162, + "n_panel": 14943, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 1000000.0, + "uncertainty_aversion": 0.0, + "weighted_pd_effective": 0.17, + "weighted_pd_point": 0.08096395060564901 + }, + "selected_policy": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "gamma": 0.5, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "selection_audit": { + "endpoint_budget_cap": 0.28, + "min_budget_utilization": 0.999, + "n_eligible": 5, + "n_total": 9, + "outcome_columns_used": 0, + "selected_candidate_id": "linear-005", + "selection_rule": "max_expected_objective_under_deterministic_endpoint_screen", + "statistical_assumption_columns_used": 0 + }, + "selector_columns": [ + "candidate_id", + "risk_tolerance", + "gamma", + "uncertainty_aversion", + "policy_mode", + "delta_cap_quantile", + "tail_focus_quantile", + "min_budget_utilization", + "pd_cap_slack_penalty", + "solver_status", + "objective_risk_mode", + "expected_objective", + "n_panel", + "n_funded", + "total_allocated", + "weighted_pd_point", + "weighted_pd_effective", + "gamma_cp", + "gamma_internalized", + "gamma_residual", + "endpoint_budget", + "markov_loss_threshold", + "effective_pd_cap_slack" + ], + "selector_forbidden_columns_present": [], + "selector_input_columns": [ + "id", + "loan_amnt", + "purpose", + "grade", + "term", + "verification_status", + "issue_d", + "_pd_point", + "_pd_low", + "_pd_high", + "_loan_amount", + "_int_rate" + ], + "source_commit": "e0daf55685988408bc68f5b65df0298174f1a516" +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json new file mode 100644 index 0000000..08f6f0b --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json @@ -0,0 +1,176 @@ +{ + "bootstrap_return_interval": { + "bootstrap_unit": "origination_month", + "n_draws": 5000, + "n_units": 31, + "p025": 163421.1394838545, + "p975": 193551.6460089551 + }, + "claim_boundary": "The final tagged rule selects among nine round-number policies on November 2017 without exposing the selector to outcomes or assumption-conditional statistics. December 2017 independently replays the outcome-free selector and audits the already-fixed decision. The audit is diagnostic rather than a selected-set coverage theorem. Earlier development inspected the static OOT corpus, so the January 2018--September 2020 evaluation remains a transparent retrospective lockbox replay, not a pristine prospective trial, causal estimate, or live-deployment guarantee.", + "exact_alpha_reference_replay": { + "high_max_abs": 6.661338147750939e-16, + "low_max_abs": 3.3306690738754696e-16, + "pass": true, + "point_max_abs": 4.440892098500626e-16, + "tolerance": 1e-12 + }, + "full_oot": { + "Gamma_CP": 0.1761021788469351, + "Gamma_internalized": 0.0880510894234675, + "Gamma_residual": 0.0880510894234675, + "endpoint_budget": 0.2580510894234676, + "expected_objective": 168271.56287282018, + "markov_loss_threshold": 0.5742788554403055, + "markov_tail_probability_bound": 0.31622776601683794, + "n_candidates": 276869, + "n_funded": 308, + "observed_accounting_bound": 0.2949260894234676, + "realized_return": 179327.5851322598, + "total_allocated": 1000000.0, + "weighted_default_rate": 0.039375, + "weighted_miscoverage": 0.036875, + "weighted_pd_effective": 0.17, + "weighted_pd_point": 0.0819489105765324 + }, + "generated_at_utc": "2026-07-10T04:24:39.860761+00:00", + "paper_tables": { + "alpha": [ + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex" + ], + "baseline": [ + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex" + ], + "bootstrap": [ + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex" + ], + "grade": [ + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex" + ], + "selector": [ + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex" + ], + "temporal": [ + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex" + ] + }, + "point_pd_contrast": { + "endpoint_budget": 0.9213174385285344, + "markov_loss_threshold": 1.2375452045453723, + "realized_return": 196369.14000000004, + "selected_default_reduction": 0.07902500000000001, + "selected_return_cost": 17041.554867740255, + "selected_return_cost_pct": 0.0867832637436832, + "selected_threshold_reduction": 0.6632663491050668, + "weighted_default_rate": 0.1184, + "weighted_miscoverage": 0.0419 + }, + "retired_active_claims": [ + "alpha01 intervals obtained by cross-family average-width scaling", + "8/8 approximate alpha-grid pass as a headline certificate", + "50,010-policy frontier as the active selector", + "0.345084 Markov threshold", + "capped_blended_uncertainty with delta_cap_quantile=0.975", + "OOT-outcome-selected portfolio hyperparameters", + "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger" + ], + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7", + "schema_version": "2026-07-09.7", + "selected_policy": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "gamma": 0.5, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "selection_protocol": { + "calibration_audit": { + "claim_boundary": "Independent post-selection decision audit; not a selected-set coverage theorem.", + "matched_point_pd": { + "endpoint_budget": 0.8880705762941006, + "n_funded": 169, + "observed_accounting_bound": 0.9463705762941006, + "realized_return": 89732.35130590109, + "weighted_default_rate": 0.18565000000000004, + "weighted_miscoverage": 0.058300000000000005 + }, + "more_conservative_policy": { + "endpoint_budget": 0.20350382300175537, + "n_funded": 191, + "observed_accounting_bound": 0.3380288230017554, + "realized_return": 38379.50087450525, + "weighted_default_rate": 0.15525, + "weighted_miscoverage": 0.134525 + }, + "n_candidates": 20695, + "outcome_free_selected_candidate_id": "linear-005", + "period": "2017-12", + "same_policy_selected": true, + "selected_policy": { + "endpoint_budget": 0.2620820323280323, + "n_funded": 193, + "observed_accounting_bound": 0.3870070323280323, + "realized_return": 53313.04791145165, + "weighted_default_rate": 0.14565, + "weighted_miscoverage": 0.12492500000000001 + } + }, + "calibration_metadata": { + "audit_period": "2017-12", + "audit_rows": 20695, + "calibration_fit_rows": 142550, + "calibration_holdout_end": "2017-12-01", + "calibration_holdout_rows": 35638, + "calibration_holdout_start": "2017-11-01", + "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", + "outcomes_isolated_until_post_selection_audit": true, + "partition": "score_decile_mondrian", + "selection_period": "2017-11", + "selection_rows": 14943, + "target_alpha": 0.1, + "used_alpha": 0.095 + }, + "endpoint_budget_cap": 0.28, + "endpoint_cap_stability": { + "cap_lower_inclusive": 0.25903604939435104, + "cap_upper_exclusive": 0.29049078888716334, + "declared_endpoint_budget_cap": 0.28, + "margin_to_lower_boundary": 0.020963950605648984, + "margin_to_upper_boundary": 0.010490788887163316, + "selected_candidate_id": "linear-005", + "selected_endpoint_budget": 0.25903604939435104 + }, + "min_budget_utilization": 0.999, + "n_eligible": 5, + "n_total": 9, + "outcome_columns_used": 0, + "selected_candidate_id": "linear-005", + "selection_rule": "max_expected_objective_under_deterministic_endpoint_screen", + "selector_forbidden_columns_present": [], + "selector_input_columns": [ + "id", + "loan_amnt", + "purpose", + "grade", + "term", + "verification_status", + "issue_d", + "_pd_point", + "_pd_low", + "_pd_high", + "_loan_amount", + "_int_rate" + ], + "statistical_assumption_columns_used": 0 + }, + "status": "active_ijds_policy" +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json new file mode 100644 index 0000000..f388671 --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json @@ -0,0 +1,149 @@ +{ + "allocation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6\\portfolio\\calibration_selected_policy_full_oot_allocations.parquet", + "calibration_metadata": { + "calibration_fit_rows": 142550, + "calibration_selection_end": "2017-12-01", + "calibration_selection_rows": 35638, + "calibration_selection_start": "2017-11-01", + "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", + "partition": "score_decile_mondrian", + "target_alpha": 0.1, + "used_alpha": 0.095 + }, + "claim_boundary": "The final tagged policy rule ranks nine round-number candidates on the calibration development block without reading default, realized-return, or other outcome-derived selector columns, and then freezes the selected policy before OOT evaluation. Conformal endpoints themselves use calibration labels, as required. Earlier project development inspected this static OOT corpus, so the evaluation is a transparent retrospective lockbox replay rather than a pristine prospective trial, causal estimate, or live-deployment guarantee.", + "config_path": "configs\\experiments\\champion_reopen_ijds_calibration_selected_simple90_v6.yaml", + "config_sha256": "cddf7daf67c91b7715e676b8a9e8dcba08e8d3a9499b8fe7cdffe51cfe690646", + "contrasts": { + "2020+": { + "default_delta_vs_incumbent": -0.010000000000000009, + "default_delta_vs_point": 0.06687499999999999, + "return_cost_vs_point": 118939.59578289572, + "return_delta_vs_incumbent": 16526.81992213076, + "selected_markov_threshold": 0.580689837874186, + "selected_realized_return": 99689.53961659266, + "selected_weighted_outcome": 0.08377499999999999, + "threshold_delta_vs_point": -0.5969627812854879 + }, + "full_oot": { + "default_delta_vs_incumbent": 0.003500000000000003, + "default_delta_vs_point": -0.07902500000000001, + "return_cost_vs_point": 17041.554867740255, + "return_delta_vs_incumbent": 6388.080277992645, + "selected_markov_threshold": 0.5742788554403055, + "selected_realized_return": 179327.5851322598, + "selected_weighted_outcome": 0.03937500000000001, + "threshold_delta_vs_point": -0.663266349105067 + } + }, + "design": { + "alpha": 0.1, + "budget": 1000000.0, + "combine_years_from": 2020, + "lgd": 0.45, + "markov_threshold_cap": 0.6, + "max_concentration": 0.25, + "period_order": [ + "2018H1", + "2018H2", + "2019H1", + "2019H2", + "2020+" + ], + "selection_min_budget_utilization": 0.999, + "selection_rule": "maximize expected point-PD objective on the calibration holdout under a 0.60 endpoint-plus-Markov screen, the effective-PD cap, and full budget use" + }, + "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6\\portfolio\\calibration_selected_policy_oot_evaluation.csv", + "generated_at_utc": "2026-07-10T02:32:57.168718+00:00", + "grid_size": 9, + "incumbent_policy": { + "candidate_id": "linear-006", + "delta_cap_quantile": 1.0, + "gamma": 0.75, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "recipe": { + "partition": "score_decile_mondrian", + "partition_probability_source": "calibrated", + "reference_target_alpha": 0.1, + "reference_used_alpha": 0.095 + }, + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6", + "schema_version": "2026-07-09.6", + "selected_calibration_metrics": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "effective_pd_cap_slack": 0.0, + "endpoint_budget": 0.26104699571114653, + "expected_objective": 110346.16233639097, + "gamma": 0.5, + "gamma_cp": 0.18209399142229304, + "gamma_internalized": 0.09104699571114652, + "gamma_residual": 0.09104699571114651, + "markov_loss_threshold": 0.5772747617279845, + "min_budget_utilization": 0.0, + "n_funded": 211, + "n_panel": 35638, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 1000000.0, + "uncertainty_aversion": 0.0, + "weighted_pd_effective": 0.17, + "weighted_pd_point": 0.07895300428885349 + }, + "selected_policy": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "gamma": 0.5, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "selection_audit": { + "markov_threshold_cap": 0.6, + "min_budget_utilization": 0.999, + "n_eligible": 5, + "n_total": 9, + "outcome_columns_used": 0, + "selected_candidate_id": "linear-005", + "selection_rule": "max_expected_objective_under_ex_ante_screen" + }, + "selector_columns": [ + "candidate_id", + "risk_tolerance", + "gamma", + "uncertainty_aversion", + "policy_mode", + "delta_cap_quantile", + "tail_focus_quantile", + "min_budget_utilization", + "pd_cap_slack_penalty", + "solver_status", + "objective_risk_mode", + "expected_objective", + "n_panel", + "n_funded", + "total_allocated", + "weighted_pd_point", + "weighted_pd_effective", + "gamma_cp", + "gamma_internalized", + "gamma_residual", + "endpoint_budget", + "markov_loss_threshold", + "effective_pd_cap_slack" + ], + "selector_forbidden_columns_present": [], + "source_commit": "4134a6d32156fb02d7c7a2787383f20caea315eb" +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json new file mode 100644 index 0000000..9d3491b --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json @@ -0,0 +1,114 @@ +{ + "bootstrap_return_interval": { + "n_draws": 5000, + "p025": 162706.17200230644, + "p975": 193924.73990027257 + }, + "claim_boundary": "The final tagged policy rule ranks nine round-number candidates on the calibration development block without reading default, realized-return, or other outcome-derived selector columns, and then freezes the selected policy before OOT evaluation. Conformal endpoints themselves use calibration labels, as required. Earlier project development inspected this static OOT corpus, so the evaluation is a transparent retrospective lockbox replay rather than a pristine prospective trial, causal estimate, or live-deployment guarantee.", + "exact_alpha_reference_replay": { + "high_max_abs": 6.661338147750939e-16, + "low_max_abs": 3.3306690738754696e-16, + "pass": true, + "point_max_abs": 4.440892098500626e-16, + "tolerance": 1e-12 + }, + "full_oot": { + "Gamma_CP": 0.1761021788469351, + "Gamma_internalized": 0.0880510894234675, + "Gamma_residual": 0.0880510894234675, + "endpoint_budget": 0.2580510894234676, + "expected_objective": 168271.56287282018, + "markov_loss_threshold": 0.5742788554403055, + "markov_tail_probability_bound": 0.31622776601683794, + "n_candidates": 276869, + "n_funded": 308, + "observed_accounting_bound": 0.2949260894234676, + "realized_return": 179327.5851322598, + "total_allocated": 1000000.0, + "weighted_default_rate": 0.039375, + "weighted_miscoverage": 0.036875, + "weighted_pd_effective": 0.17, + "weighted_pd_point": 0.0819489105765324 + }, + "generated_at_utc": "2026-07-10T02:32:57.168718+00:00", + "paper_tables": { + "alpha": [ + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex" + ], + "baseline": [ + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex" + ], + "bootstrap": [ + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex" + ], + "grade": [ + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex" + ], + "selector": [ + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex" + ], + "temporal": [ + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex" + ] + }, + "point_pd_contrast": { + "endpoint_budget": 0.9213174385285344, + "markov_loss_threshold": 1.2375452045453723, + "realized_return": 196369.14000000004, + "selected_default_reduction": 0.07902500000000001, + "selected_return_cost": 17041.554867740255, + "selected_return_cost_pct": 0.0867832637436832, + "selected_threshold_reduction": 0.6632663491050668, + "weighted_default_rate": 0.1184, + "weighted_miscoverage": 0.0419 + }, + "retired_active_claims": [ + "alpha01 intervals obtained by cross-family average-width scaling", + "8/8 approximate alpha-grid pass as a headline certificate", + "50,010-policy frontier as the active selector", + "0.345084 Markov threshold", + "capped_blended_uncertainty with delta_cap_quantile=0.975", + "OOT-outcome-selected portfolio hyperparameters", + "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger" + ], + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6", + "schema_version": "2026-07-09.6", + "selected_policy": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "gamma": 0.5, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "selection_protocol": { + "calibration_metadata": { + "calibration_fit_rows": 142550, + "calibration_selection_end": "2017-12-01", + "calibration_selection_rows": 35638, + "calibration_selection_start": "2017-11-01", + "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", + "partition": "score_decile_mondrian", + "target_alpha": 0.1, + "used_alpha": 0.095 + }, + "markov_threshold_cap": 0.6, + "min_budget_utilization": 0.999, + "n_eligible": 5, + "n_total": 9, + "outcome_columns_used": 0, + "selected_candidate_id": "linear-005", + "selection_rule": "max_expected_objective_under_ex_ante_screen", + "selector_forbidden_columns_present": [] + }, + "status": "active_ijds_policy" +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json new file mode 100644 index 0000000..34a0363 --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json @@ -0,0 +1,358 @@ +{ + "generated_at_utc": "2026-07-09T22:29:22.353666+00:00", + "source_run_tags": [ + "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", + "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", + "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" + ], + "selection_rule": { + "eligible": "all-alpha pass and nonnegative return_floor_surplus", + "dedupe_key": "semantic_policy_key", + "dedupe_semantics": "duplicate semantic policies across refinement runs have identical metrics; one representative row is retained for the consolidated table", + "body_selection": "highest realized return among eligible finite-grid policies with exact Markov_threshold <= 0.35; falls back to the legacy balanced normalized return/bound/V score only if no eligible policy exists under that declared threshold", + "bound_semantics": "Markov_threshold = weighted endpoint budget B_u + sqrt(alpha); Markov_cap = tau + residual endpoint premium + solver slack + sqrt(alpha). The exact threshold drives selection.", + "caps": [ + 0.3, + 0.32, + 0.345, + 0.36, + 0.45 + ], + "role_semantics": "finite-grid frontier roles, not continuous optima" + }, + "counts": { + "raw_rows": 51678, + "deduped_semantic_policies": 50010, + "duplicate_rows_removed": 1668, + "eligible_all_alpha_return_floor_policies": 27508, + "nonpass_or_below_floor_policies": 22502 + }, + "certificate_semantics_audit": { + "status": "corrected_from_existing_exact_bound_evaluations", + "selection_metric": "alpha01_markov_loss_threshold", + "legacy_metric": "tau + (1 - gamma) * Gamma_CP + sqrt(alpha)", + "material_difference_tolerance": 1e-05, + "materially_changed_policies": 10423, + "materially_understated_policies": 2866, + "maximum_legacy_understatement": 0.2413235, + "legacy_under_0_50_excluded_by_exact_threshold": 716, + "body_selection_unchanged": true, + "affected_policy_modes": [ + "segment_relative_tail_blended_uncertainty", + "tail_blended_uncertainty" + ] + }, + "by_run": [ + { + "run_label": "expanded", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", + "rows": 3258, + "all_alpha_passers": 3071, + "all_alpha_pass_rate": 0.942602824, + "best_return": 223346.5525, + "min_markov_threshold": 0.335785 + }, + { + "run_label": "micro", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", + "rows": 2949, + "all_alpha_passers": 2949, + "all_alpha_pass_rate": 1.0, + "best_return": 223369.907874, + "min_markov_threshold": 0.328799 + }, + { + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "rows": 4407, + "all_alpha_passers": 4406, + "all_alpha_pass_rate": 0.999773088, + "best_return": 223458.135875, + "min_markov_threshold": 0.316713 + }, + { + "run_label": "bound_closure", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "rows": 1653, + "all_alpha_passers": 1653, + "all_alpha_pass_rate": 1.0, + "best_return": 213593.775, + "min_markov_threshold": 0.298369 + }, + { + "run_label": "bound_floor", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", + "rows": 2343, + "all_alpha_passers": 2343, + "all_alpha_pass_rate": 1.0, + "best_return": 213593.775, + "min_markov_threshold": 0.284839 + }, + { + "run_label": "bound_terminal", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "rows": 37068, + "all_alpha_passers": 37068, + "all_alpha_pass_rate": 1.0, + "best_return": 213593.775, + "min_markov_threshold": 0.267672 + } + ], + "rows": [ + { + "role": "minimum Markov-threshold endpoint", + "run_label": "bound_terminal", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "local_candidate_id": 10662, + "family": "claim_bound_terminal_ultra_low_cap", + "anchor_rank": 219, + "source_reason": "terminal_cap_threshold_search", + "risk_tolerance": 0.16825, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.95, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.6, + "return": 170467.268819, + "return_floor_surplus": 2.728819, + "Gamma_CP": 0.095719, + "Gamma_internalized": 0.090933, + "Gamma_residual": 0.004786, + "V": 0.031875, + "endpoint_budget": 0.173036, + "endpoint_budget_upper": 0.173036, + "Markov_threshold": 0.273036, + "Markov_cap": 0.273036, + "alpha_pass": "8/8", + "n_funded_mean": 311.125, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.95,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.16825,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.6}" + }, + { + "role": "body/default balanced return-bound point", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 131, + "family": "claim_micro_ext_body_cap345", + "anchor_rank": 219, + "source_reason": "candidate37_205_body_cap345_extension", + "risk_tolerance": 0.1715, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.5475, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.05, + "return": 184832.475845, + "return_floor_surplus": 14367.935845, + "Gamma_CP": 0.162616, + "Gamma_internalized": 0.089032, + "Gamma_residual": 0.073584, + "V": 0.03535, + "endpoint_budget": 0.245084, + "endpoint_budget_upper": 0.245084, + "Markov_threshold": 0.345084, + "Markov_cap": 0.345084, + "alpha_pass": "8/8", + "n_funded_mean": 320.5, + "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5475,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1715,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.05}" + }, + { + "role": "lowest realized V return-bound point", + "run_label": "bound_terminal", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "local_candidate_id": 5504, + "family": "claim_bound_terminal_ultra_low_cap", + "anchor_rank": 219, + "source_reason": "terminal_cap_threshold_search", + "risk_tolerance": 0.1675, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.925, + "delta_cap_quantile": 0.95, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.6, + "return": 171006.195983, + "return_floor_surplus": 541.655983, + "Gamma_CP": 0.09719, + "Gamma_internalized": 0.089901, + "Gamma_residual": 0.007289, + "V": 0.031875, + "endpoint_budget": 0.174789, + "endpoint_budget_upper": 0.174789, + "Markov_threshold": 0.274789, + "Markov_cap": 0.274789, + "alpha_pass": "8/8", + "n_funded_mean": 311.125, + "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.925,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1675,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.6}" + }, + { + "role": "max-return economic endpoint", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 4041, + "family": "claim_micro_ext_economic_endpoint", + "anchor_rank": 96, + "source_reason": "candidate2122_economic_endpoint_extension", + "risk_tolerance": 0.156875, + "policy_mode": "tail_blended_uncertainty", + "gamma": 0.445, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 0.925, + "uncertainty_aversion": 0.1375, + "return": 223458.135875, + "return_floor_surplus": 52993.595875, + "Gamma_CP": 0.457438, + "Gamma_internalized": 0.017256, + "Gamma_residual": 0.440181, + "V": 0.069575, + "endpoint_budget": 0.597056, + "endpoint_budget_upper": 0.597056, + "Markov_threshold": 0.697056, + "Markov_cap": 0.697056, + "alpha_pass": "8/8", + "n_funded_mean": 239.625, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.445,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"tail_blended_uncertainty\",\"risk_tolerance\":0.156875,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":0.925,\"uncertainty_aversion\":0.1375}" + }, + { + "role": "highest return under threshold<=0.3", + "run_label": "bound_closure", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "local_candidate_id": 992, + "family": "claim_bound_closure_low_cap", + "anchor_rank": 219, + "source_reason": "micro_ext_min_markov_cap_endpoint_closure", + "risk_tolerance": 0.16975, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.75, + "delta_cap_quantile": 0.95, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.35, + "return": 174552.51361, + "return_floor_surplus": 4087.97361, + "Gamma_CP": 0.120988, + "Gamma_internalized": 0.090741, + "Gamma_residual": 0.030247, + "V": 0.035875, + "endpoint_budget": 0.199997, + "endpoint_budget_upper": 0.199997, + "Markov_threshold": 0.299997, + "Markov_cap": 0.299997, + "alpha_pass": "8/8", + "n_funded_mean": 306.625, + "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.75,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.16975,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.35}" + }, + { + "role": "highest return under threshold<=0.32", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 2021, + "family": "claim_micro_ext_bound_tight", + "anchor_rank": 219, + "source_reason": "candidate949_bound_tight_extension", + "risk_tolerance": 0.1705, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.645, + "delta_cap_quantile": 0.95, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.2375, + "return": 179436.119445, + "return_floor_surplus": 8971.579445, + "Gamma_CP": 0.139182, + "Gamma_internalized": 0.089773, + "Gamma_residual": 0.04941, + "V": 0.035875, + "endpoint_budget": 0.21991, + "endpoint_budget_upper": 0.21991, + "Markov_threshold": 0.31991, + "Markov_cap": 0.31991, + "alpha_pass": "8/8", + "n_funded_mean": 310.125, + "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.645,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1705,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.2375}" + }, + { + "role": "highest return under threshold<=0.345", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 512, + "family": "claim_micro_ext_body_cap345", + "anchor_rank": 219, + "source_reason": "candidate37_205_body_cap345_extension", + "risk_tolerance": 0.17225, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.5525, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0375, + "return": 184800.413581, + "return_floor_surplus": 14335.873581, + "Gamma_CP": 0.162562, + "Gamma_internalized": 0.089816, + "Gamma_residual": 0.072747, + "V": 0.03535, + "endpoint_budget": 0.244997, + "endpoint_budget_upper": 0.244997, + "Markov_threshold": 0.344997, + "Markov_cap": 0.344997, + "alpha_pass": "8/8", + "n_funded_mean": 321.0, + "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5525,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17225,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.0375}" + }, + { + "role": "highest return under threshold<=0.36", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 3212, + "family": "claim_micro_ext_cap036_return", + "anchor_rank": 219, + "source_reason": "candidate1975_cap036_return_extension", + "risk_tolerance": 0.17575, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.525, + "delta_cap_quantile": 0.95, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.075, + "return": 186050.727749, + "return_floor_surplus": 15586.187749, + "Gamma_CP": 0.1746, + "Gamma_internalized": 0.091665, + "Gamma_residual": 0.082935, + "V": 0.03775, + "endpoint_budget": 0.258685, + "endpoint_budget_upper": 0.258685, + "Markov_threshold": 0.358685, + "Markov_cap": 0.358685, + "alpha_pass": "8/8", + "n_funded_mean": 318.75, + "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.525,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17575,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.075}" + }, + { + "role": "highest return under threshold<=0.45", + "run_label": "expanded", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", + "local_candidate_id": 979, + "family": "bound_efficient_local", + "anchor_rank": 223, + "source_reason": "rank219_rank223_bound_frontier", + "risk_tolerance": 0.185, + "policy_mode": "blended_uncertainty", + "gamma": 0.35, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.15, + "return": 198693.277519, + "return_floor_surplus": 28228.737519, + "Gamma_CP": 0.252323, + "Gamma_internalized": 0.088313, + "Gamma_residual": 0.16401, + "V": 0.0456, + "endpoint_budget": 0.34901, + "endpoint_budget_upper": 0.34901, + "Markov_threshold": 0.44901, + "Markov_cap": 0.44901, + "alpha_pass": "8/8", + "n_funded_mean": 310.875, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.35,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"blended_uncertainty\",\"risk_tolerance\":0.185,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.15}" + } + ] +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json new file mode 100644 index 0000000..7f7476b --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json @@ -0,0 +1,198 @@ +{ + "generated_at_utc": "2026-07-09T23:10:36.956639+00:00", + "source_frontier_path": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\models\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2\\portfolio\\pool93_ijds_consolidated_frontier.json", + "source_run_tags": [ + "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", + "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", + "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" + ], + "counts": { + "raw_rows": 51678, + "deduped_semantic_policies": 50010, + "duplicate_rows_removed": 1668, + "eligible_all_alpha_return_floor_policies": 27508, + "nonpass_or_below_floor_policies": 22502 + }, + "selection_rule": { + "eligible": "all-alpha pass and nonnegative return_floor_surplus", + "dedupe_key": "semantic_policy_key", + "dedupe_semantics": "duplicate semantic policies across refinement runs have identical metrics; one representative row is retained for the consolidated table", + "body_selection": "highest realized return among eligible finite-grid policies with exact Markov_threshold <= 0.35; falls back to the legacy balanced normalized return/bound/V score only if no eligible policy exists under that declared threshold", + "bound_semantics": "Markov_threshold = weighted endpoint budget B_u + sqrt(alpha); Markov_cap = tau + residual endpoint premium + solver slack + sqrt(alpha). The exact threshold drives selection.", + "caps": [ + 0.3, + 0.32, + 0.345, + 0.36, + 0.45 + ], + "role_semantics": "finite-grid frontier roles, not continuous optima" + }, + "certificate_semantics_audit": { + "status": "corrected_from_existing_exact_bound_evaluations", + "selection_metric": "alpha01_markov_loss_threshold", + "legacy_metric": "tau + (1 - gamma) * Gamma_CP + sqrt(alpha)", + "material_difference_tolerance": 1e-05, + "materially_changed_policies": 10423, + "materially_understated_policies": 2866, + "maximum_legacy_understatement": 0.2413235, + "legacy_under_0_50_excluded_by_exact_threshold": 716, + "body_selection_unchanged": true, + "affected_policy_modes": [ + "segment_relative_tail_blended_uncertainty", + "tail_blended_uncertainty" + ] + }, + "claim_hierarchy": { + "status": "final", + "paper_body_candidate": "body/default balanced return-bound point", + "paper_body_claim": "The selected pool93 body point is the highest-return eligible finite-grid policy under the declared exact Markov-threshold ceiling and passes all eight predeclared alpha checks.", + "appendix_frontier_candidates": [ + "minimum Markov-threshold endpoint", + "highest return under threshold<=0.345", + "highest return under threshold<=0.36", + "max-return economic endpoint" + ], + "do_not_claim": [ + "continuous-region optimality beyond the evaluated finite policy grid", + "nominal funded-set alpha coverage when V(alpha) exceeds alpha", + "prospective live-selection validity from retrospective OOT selection", + "pool93-specific row-level tail/CVaR dominance unless regenerated from the promoted allocation" + ], + "promotion_gate": [ + "consolidated frontier generated from completed exact runs", + "semantic-policy deduplication applied", + "policy-aware endpoint decomposition applied to every alpha=0.01 row", + "selected body point passes 8/8 alpha checks", + "zero realized risk-tolerance excess at alpha=0.01", + "return exceeds declared return floor", + "A35 frontier, A36 funded-set grade audit, and A40 matched baseline are regenerated from retained artifacts" + ] + }, + "selected_candidates": { + "paper_body": { + "role": "body/default balanced return-bound point", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 131, + "family": "claim_micro_ext_body_cap345", + "anchor_rank": 219, + "source_reason": "candidate37_205_body_cap345_extension", + "risk_tolerance": 0.1715, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.5475, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.05, + "return": 184832.475845, + "return_floor_surplus": 14367.935845, + "Gamma_CP": 0.162616, + "Gamma_internalized": 0.089032, + "Gamma_residual": 0.073584, + "V": 0.03535, + "endpoint_budget": 0.245084, + "endpoint_budget_upper": 0.245084, + "Markov_threshold": 0.345084, + "Markov_cap": 0.345084, + "alpha_pass": "8/8", + "n_funded_mean": 320.5, + "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5475,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1715,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.05}" + }, + "strict_threshold_leq_0_345": { + "role": "highest return under threshold<=0.345", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 512, + "family": "claim_micro_ext_body_cap345", + "anchor_rank": 219, + "source_reason": "candidate37_205_body_cap345_extension", + "risk_tolerance": 0.17225, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.5525, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0375, + "return": 184800.413581, + "return_floor_surplus": 14335.873581, + "Gamma_CP": 0.162562, + "Gamma_internalized": 0.089816, + "Gamma_residual": 0.072747, + "V": 0.03535, + "endpoint_budget": 0.244997, + "endpoint_budget_upper": 0.244997, + "Markov_threshold": 0.344997, + "Markov_cap": 0.344997, + "alpha_pass": "8/8", + "n_funded_mean": 321.0, + "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5525,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17225,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.0375}" + }, + "minimum_markov_threshold_endpoint": { + "role": "minimum Markov-threshold endpoint", + "run_label": "bound_terminal", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "local_candidate_id": 10662, + "family": "claim_bound_terminal_ultra_low_cap", + "anchor_rank": 219, + "source_reason": "terminal_cap_threshold_search", + "risk_tolerance": 0.16825, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.95, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.6, + "return": 170467.268819, + "return_floor_surplus": 2.728819, + "Gamma_CP": 0.095719, + "Gamma_internalized": 0.090933, + "Gamma_residual": 0.004786, + "V": 0.031875, + "endpoint_budget": 0.173036, + "endpoint_budget_upper": 0.173036, + "Markov_threshold": 0.273036, + "Markov_cap": 0.273036, + "alpha_pass": "8/8", + "n_funded_mean": 311.125, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.95,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.16825,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.6}" + }, + "max_return_economic_endpoint": { + "role": "max-return economic endpoint", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 4041, + "family": "claim_micro_ext_economic_endpoint", + "anchor_rank": 96, + "source_reason": "candidate2122_economic_endpoint_extension", + "risk_tolerance": 0.156875, + "policy_mode": "tail_blended_uncertainty", + "gamma": 0.445, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 0.925, + "uncertainty_aversion": 0.1375, + "return": 223458.135875, + "return_floor_surplus": 52993.595875, + "Gamma_CP": 0.457438, + "Gamma_internalized": 0.017256, + "Gamma_residual": 0.440181, + "V": 0.069575, + "endpoint_budget": 0.597056, + "endpoint_budget_upper": 0.597056, + "Markov_threshold": 0.697056, + "Markov_cap": 0.697056, + "alpha_pass": "8/8", + "n_funded_mean": 239.625, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.445,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"tail_blended_uncertainty\",\"risk_tolerance\":0.156875,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":0.925,\"uncertainty_aversion\":0.1375}" + } + }, + "paper_artifacts": { + "frontier_table_csv": "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv", + "frontier_table_tex": "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex", + "funded_grade_audit_csv": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", + "funded_grade_audit_tex": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex", + "point_baseline_csv": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv", + "point_baseline_tex": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex", + "point_baseline_audit": "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json" + } +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json new file mode 100644 index 0000000..d51dbe5 --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json @@ -0,0 +1,86 @@ +{ + "schema_version": "2026-07-09.1", + "generated_at_utc": "2026-07-09T22:41:29.886155+00:00", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2", + "comparison": "matched point-PD two-stage LP versus selected CRPTO", + "fixed_design": { + "candidate_universe": 276869, + "budget": 1000000.0, + "risk_tolerance": 0.1715, + "max_concentration": 0.25, + "alpha": 0.01 + }, + "point_pd_baseline": { + "solver_status": "Optimal", + "realized_return": 196369.14000000004, + "total_allocated": 1000000.0, + "expected_return_gross": 283248.8175, + "expected_loss_point": 69229.66551833821, + "expected_return_net_point": 214019.1519816618, + "certificate": { + "alpha": 0.01, + "risk_tolerance": 0.1715, + "n_funded": 225, + "weighted_outcome": 0.11840000000000002, + "weighted_miscoverage": 0.11840000000000002, + "weighted_coverage": 0.8815999999999999, + "empirical_coverage_funded": 0.8844444444444445, + "weighted_pd_point": 0.1538437011518627, + "weighted_pd_effective": 0.1538437011518627, + "endpoint_budget": 0.680579296391784, + "gamma_cp": 0.5267355952399213, + "gamma_internalized": 0.0, + "gamma_residual": 0.5267355952399213, + "effective_constraint_slack": 0.017656298848137325, + "effective_constraint_excess": 0.0, + "realized_risk_tolerance_excess": 0.0, + "sqrt_alpha": 0.1, + "endpoint_budget_upper": 0.6982355952399213, + "markov_loss_threshold": 0.780579296391784, + "markov_loss_cap": 0.7982355952399213 + } + }, + "selected_crpto": { + "solver_status": "frozen_selected_allocation", + "realized_return": 184832.47584455396, + "total_allocated": 1000000.0, + "expected_return_gross": 208153.285844554, + "expected_loss_point": 37110.41439596723, + "expected_return_net_point": 171042.87144858675, + "certificate": { + "alpha": 0.01, + "risk_tolerance": 0.1715, + "n_funded": 314, + "weighted_outcome": 0.03535, + "weighted_miscoverage": 0.03535, + "weighted_coverage": 0.96465, + "empirical_coverage_funded": 0.9426751592356688, + "weighted_pd_point": 0.08246758754659385, + "weighted_pd_effective": 0.1715, + "endpoint_budget": 0.24508386600030369, + "gamma_cp": 0.1626162784537099, + "gamma_internalized": 0.08903241245340617, + "gamma_residual": 0.07358386600030371, + "effective_constraint_slack": 0.0, + "effective_constraint_excess": 0.0, + "realized_risk_tolerance_excess": 0.0, + "sqrt_alpha": 0.1, + "endpoint_budget_upper": 0.24508386600030374, + "markov_loss_threshold": 0.3450838660003037, + "markov_loss_cap": 0.3450838660003037 + } + }, + "contrasts": { + "realized_return_cost": 11536.66415544608, + "realized_return_cost_pct": 5.874988379256577, + "weighted_default_rate_reduction": 0.08305000000000001, + "weighted_miscoverage_reduction": 0.08305000000000001, + "markov_threshold_reduction": 0.4354954303914803 + }, + "claim_boundary": "Frozen OOT matched-policy audit; it quantifies a return-risk trade-off and does not establish causal, prospective or universal dominance.", + "outputs": { + "point_funded_rows": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2\\portfolio\\pool93_point_pd_baseline_alpha01.parquet", + "table_csv": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\reports\\crpto\\experiments\\champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2\\crpto_tableA40_pool93_point_baseline.csv", + "table_tex": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\reports\\crpto\\experiments\\champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2\\crpto_tableA40_pool93_point_baseline.tex" + } +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json new file mode 100644 index 0000000..ef2eb7a --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json @@ -0,0 +1,244 @@ +{ + "alpha_mapping": "proportional frozen conservative ratio alpha_used/alpha_target from the selected 90% recipe", + "alpha_summaries": [ + { + "avg_width": 0.9882149910656284, + "coverage_gap": 0.006720470691915725, + "empirical_coverage": 0.9967204706919157, + "group_quantiles": { + "score_q00": 4.620892417510161, + "score_q01": 2.9770143672127003, + "score_q02": 2.2280905155147157, + "score_q03": 1.7679793270133157, + "score_q04": 1.379561550348839 + }, + "high_endpoint_at_one_rate": 0.9354243342519386, + "high_endpoint_mean": 0.9883685381679577, + "high_endpoint_min": 0.28857911767290634, + "high_endpoint_p01": 0.6715297528015821, + "high_endpoint_p10": 1.0, + "median_width": 1.0, + "min_grade_coverage": 0.9722703639514731, + "min_partition_coverage": 0.989628743645835, + "partition_count": 5, + "target_alpha": 0.01, + "target_coverage": 0.99, + "used_alpha": 0.0095 + }, + { + "avg_width": 0.9697789837688854, + "coverage_gap": 0.01847830562468178, + "empirical_coverage": 0.9884783056246818, + "group_quantiles": { + "score_q00": 3.642519971959776, + "score_q01": 2.8072631501367455, + "score_q02": 2.1737923639215784, + "score_q03": 1.7387907144323151, + "score_q04": 1.3580168082764283 + }, + "high_endpoint_at_one_rate": 0.8323141991338864, + "high_endpoint_mean": 0.9699657056860221, + "high_endpoint_min": 0.22825533153683755, + "high_endpoint_p01": 0.5334804414885259, + "high_endpoint_p10": 0.9018799702618802, + "median_width": 1.0, + "min_grade_coverage": 0.9664933564413634, + "min_partition_coverage": 0.9697990273996318, + "partition_count": 5, + "target_alpha": 0.03, + "target_coverage": 0.97, + "used_alpha": 0.028499999999999998 + }, + { + "avg_width": 0.9535799768723978, + "coverage_gap": 0.022824693266490725, + "empirical_coverage": 0.9728246932664907, + "group_quantiles": { + "score_q00": 3.233734520299044, + "score_q01": 2.6752015014256125, + "score_q02": 2.1212263259384927, + "score_q03": 1.7077473640552945, + "score_q04": 1.3388102247632694 + }, + "high_endpoint_at_one_rate": 0.7181555175913519, + "high_endpoint_mean": 0.953801858012357, + "high_endpoint_min": 0.20305073151598757, + "high_endpoint_p01": 0.47580041346523627, + "high_endpoint_p10": 0.806835962194862, + "median_width": 1.0, + "min_grade_coverage": 0.9525750124329999, + "min_partition_coverage": 0.9505830880848586, + "partition_count": 5, + "target_alpha": 0.05, + "target_coverage": 0.95, + "used_alpha": 0.0475 + }, + { + "avg_width": 0.795666199875876, + "coverage_gap": 0.025271265472118665, + "empirical_coverage": 0.9552712654721186, + "group_quantiles": { + "score_q00": 0.32438420480886127, + "score_q01": 2.567436984095836, + "score_q02": 2.0727963588460026, + "score_q03": 1.6764580225891925, + "score_q04": 1.3186561259152434 + }, + "high_endpoint_at_one_rate": 0.6135500904760013, + "high_endpoint_mean": 0.7959333620592705, + "high_endpoint_min": 0.023668097607277484, + "high_endpoint_p01": 0.06528823147517372, + "high_endpoint_p10": 0.13040211717455735, + "median_width": 1.0, + "min_grade_coverage": 0.938442835829143, + "min_partition_coverage": 0.9347822665680426, + "partition_count": 5, + "target_alpha": 0.07, + "target_coverage": 0.9299999999999999, + "used_alpha": 0.0665 + }, + { + "avg_width": 0.7888790790793794, + "coverage_gap": 0.03483560817570763, + "empirical_coverage": 0.9348356081757077, + "group_quantiles": { + "score_q00": 0.3201476773467629, + "score_q01": 2.4331722444504513, + "score_q02": 2.007709882293012, + "score_q03": 1.633544476383572, + "score_q04": 1.2899294761760491 + }, + "high_endpoint_at_one_rate": 0.5178730735474177, + "high_endpoint_mean": 0.7892193805747533, + "high_endpoint_min": 0.023406884837288408, + "high_endpoint_p01": 0.06469045329656556, + "high_endpoint_p10": 0.12941711012832322, + "median_width": 1.0, + "min_grade_coverage": 0.9267968822615554, + "min_partition_coverage": 0.9263099219620958, + "partition_count": 5, + "target_alpha": 0.1, + "target_coverage": 0.9, + "used_alpha": 0.095 + }, + { + "avg_width": 0.7831737841625059, + "coverage_gap": 0.038264594447193434, + "empirical_coverage": 0.9182645944471934, + "group_quantiles": { + "score_q00": 0.31739006163700956, + "score_q01": 2.3534507550187485, + "score_q02": 1.9655367760210574, + "score_q03": 1.6066047959404717, + "score_q04": 1.269964951004726 + }, + "high_endpoint_at_one_rate": 0.4545109781160043, + "high_endpoint_mean": 0.7835760369425744, + "high_endpoint_min": 0.023236857746810564, + "high_endpoint_p01": 0.06430135101621326, + "high_endpoint_p10": 0.12877595509528877, + "median_width": 0.9867966540367542, + "min_grade_coverage": 0.9057375416605808, + "min_partition_coverage": 0.906874767744333, + "partition_count": 5, + "target_alpha": 0.12, + "target_coverage": 0.88, + "used_alpha": 0.11399999999999999 + }, + { + "avg_width": 0.6461762449744832, + "coverage_gap": 0.03609775742318566, + "empirical_coverage": 0.8860977574231856, + "group_quantiles": { + "score_q00": 0.31337107987851737, + "score_q01": 0.4420899636645897, + "score_q02": 1.9079870902279796, + "score_q03": 1.5675398154360425, + "score_q04": 1.2412696239135141 + }, + "high_endpoint_at_one_rate": 0.33485872380078663, + "high_endpoint_mean": 0.6466929852254412, + "high_endpoint_min": 0.02298905825369661, + "high_endpoint_p01": 0.06373426874967675, + "high_endpoint_p10": 0.1278415281651632, + "median_width": 0.94425532567159, + "min_grade_coverage": 0.8631010078209027, + "min_partition_coverage": 0.8635623982976592, + "partition_count": 5, + "target_alpha": 0.15, + "target_coverage": 0.85, + "used_alpha": 0.1425 + }, + { + "avg_width": 0.6365853175970958, + "coverage_gap": 0.049380031711748096, + "empirical_coverage": 0.8493800317117481, + "group_quantiles": { + "score_q00": 0.30656504576058813, + "score_q01": 0.43573433937656825, + "score_q02": 1.8175729021078393, + "score_q03": 1.5052283270500615, + "score_q04": 1.194771966347316 + }, + "high_endpoint_at_one_rate": 0.24466805601204902, + "high_endpoint_mean": 0.6373641323567814, + "high_endpoint_min": 0.022569416686283303, + "high_endpoint_p01": 0.06277393066265638, + "high_endpoint_p10": 0.1262591020808178, + "median_width": 0.9128005695874017, + "min_grade_coverage": 0.828006532910199, + "min_partition_coverage": 0.8293756967670011, + "partition_count": 5, + "target_alpha": 0.2, + "target_coverage": 0.8, + "used_alpha": 0.19 + } + ], + "claim_boundary": "Recomputes finite-sample Mondrian quantiles for every declared alpha while freezing the selected partition, calibration fit/holdout split, score scale, and holdout-learned widening factors. It is a retrospective OOT audit, not a post-selection or live-deployment guarantee.", + "config_path": "configs\\experiments\\champion_reopen_ijds_exact_alpha_grid_v1.yaml", + "config_sha256": "c2d2af0d1c0e0f83ef8612888f9b12d423063a601f04c8c51c9680e92327ef3f", + "generated_at_utc": "2026-07-10T02:32:16.941448+00:00", + "grid_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1\\conformal\\exact_alpha_grid.parquet", + "grid_rows": 276869, + "recipe": { + "calibration_fraction": 0.75, + "fallback_mode": "score_only", + "global_rebalance_factor": 1.0, + "group_multipliers": { + "score_q00": 1.02, + "score_q01": 1.05, + "score_q02": 1.05, + "score_q03": 1.02, + "score_q04": 1.02 + }, + "min_group_size": 100, + "n_score_bins": 5, + "partition": "score_decile_mondrian", + "partition_probability_source": "calibrated", + "reference_target_alpha": 0.1, + "reference_used_alpha": 0.095, + "score_scale_family": "bernoulli_sqrt", + "temporal_segment_freq": "Q", + "temporal_segment_multipliers": {}, + "tuning_holdout_ratio": 0.2, + "tuning_random_state": 42 + }, + "reference_replay": { + "high_max_abs": 6.661338147750939e-16, + "low_max_abs": 3.3306690738754696e-16, + "pass": true, + "point_max_abs": 4.440892098500626e-16, + "tolerance": 1e-12 + }, + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1", + "schema_version": "2026-07-09.1", + "source": { + "conformal_intervals_path": "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet", + "conformal_intervals_sha256": "3784cc68a5b72cf290c9678e6cd5d30f7b794dac4c1eedaf393fe9f72cc0c3a1", + "conformal_namespace": "champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1", + "conformal_results_path": "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl", + "conformal_results_sha256": "cb8db92908739c37a11f373b9802f6b74831d8d9c26c4824dae586423d3cb07b", + "upstream_canonical_run_tag": "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" + }, + "source_commit": "4134a6d32156fb02d7c7a2787383f20caea315eb" +} diff --git a/paper/CRPTO.qmd b/paper/CRPTO.qmd index 155ad00..29ca30e 100644 --- a/paper/CRPTO.qmd +++ b/paper/CRPTO.qmd @@ -14,52 +14,47 @@ execute: # Resumen -CRPTO integra prediccion de default calibrada, intervalos conformales Mondrian y -optimizacion robusta de portafolio. La superficie IJDS activa promueve el punto -pool93 body/default: retorno robusto `$184,832.48`, `V=0.035350`, -`Gamma_CP=0.162616`, Markov cap `0.345084`, zero violation y pass `8/8` sobre -la grilla alpha declarada. La frontera A35 y el audit de composicion A36 son los -artefactos paper-facing principales para el cierre pool93; A37--A39 agregan la -repricing de tail-risk, el audit cluster-bound y el bootstrap de contribuciones -de la asignacion seleccionada. - -# Manuscrito - -Este archivo es el punto de entrada generico del manuscrito standalone. La -decision editorial activa esta en -`../configs/crpto_publication_targets.yaml`: escribir primero para **INFORMS -Journal on Data Science** y mantener **European Journal of Operational -Research** como pivote principal. - -Los borradores de trabajo son: - -- `CRPTO_ijds.qmd`: cuerpo anonimo IJDS, conceptualmente limitado a 25 paginas - excluyendo referencias y appendix. -- `supplement_ijds.qmd`: online supplement IJDS con A3--A39, robustez, - reproducibilidad, MRM y fairness. - -La version larga y sus apendices estan en `book/`. - -# Artefactos principales - -Esquema dual-tag: el body claim del paper vive en los sidecars pool93; la -cadena upstream congelada (rebaseline 2026-06-07) define el return floor -declarado. `crpto_table0_key_metrics.csv` y `final_project_promotion.json` -documentan esa cadena historica, no el body point. - -Body claim pool93 (autoritativos): - -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` (y A36--A39) - -Cadena upstream congelada (historica, return floor): - -- `models/final_project_promotion.json` -- `models/champion_portfolio_policy.json` -- `reports/crpto/tables/crpto_table0_key_metrics.csv` - -Figuras de entrada: - -- `reports/crpto/figures/crpto_fig1_journal_pipeline.png` -- `reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.png` +CRPTO convierte una PD calibrada en una decision de portafolio auditable. La +version IJDS activa reproduce exactamente el endpoint conformal al 90%, usa la +regla simple `q=(p+u)/2` con `tau=0.17`, y selecciona esa politica dentro de una +grilla `3x3` en noviembre bajo `B_u<=0.28`. Un replay outcome-free en diciembre +elige la misma politica; al abrir outcomes, la miscoverage `0.124925` muestra +que estabilidad de seleccion no equivale a cobertura del funded set. + +En 276,869 prestamos OOT, la politica financia 308 prestamos y obtiene +`$179,327.59` sobre `$1M`, con default ponderado `0.039375`, miscoverage +`0.036875` y endpoint `0.258051`. La baseline point-PD emparejada obtiene +`$196,369.14` y default `0.118400`. El costo observado de retorno es `8.678%` y +la reduccion de default es `7.9025` puntos porcentuales. El paper reporta tambien +los periodos donde point-PD domina; no afirma superioridad universal. + +# Fuentes del manuscrito + +- `CRPTO_ijds.qmd`: body anonimo y fuente narrativa activa. +- `supplement_ijds.qmd`: pruebas y evidencia A35--A40, mas diagnosticos + historicos A1--A34. +- `submission/CRPTO_ijds_submission.tex`: handoff compacto bajo `informs4`. +- `../docs/research/active_claims_2026-07-04.md`: contrato numerico vigente. + +# Evidencia activa + +- A35: replay exacto y saturacion por alpha. +- A36: selector de noviembre, estabilidad del cap y auditoria de diciembre. +- A37: evaluacion OOT total y temporal. +- A38: composicion por grado de credito. +- A39: bootstrap por mes de originacion y sensibilidad por prestamo. +- A40: politica seleccionada, blend conservador y baseline point-PD. + +La gobernanza vive en +`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. +Los artefactos historicos protegidos por `EXTRACTION_MANIFEST.json` permanecen +intactos como procedencia, no como claims activos. + +# Comandos + +```powershell +just ijds-evidence +just paper-submission +just paper-submission-official +just submission-check +``` diff --git a/paper/CRPTO_ijds.qmd b/paper/CRPTO_ijds.qmd index 364dc10..e1dd100 100644 --- a/paper/CRPTO_ijds.qmd +++ b/paper/CRPTO_ijds.qmd @@ -1,5 +1,5 @@ --- -title: "CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit Portfolio Decisions" +title: "CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios" author: "Anonymous" date: today lang: en @@ -25,1034 +25,519 @@ execute: # Abstract -Credit allocation is a data-science-for-decisions problem: calibrated default -probabilities matter only after they shape which loans are funded. We introduce -Conformal Robust Predict-Then-Optimize (CRPTO), a post-hoc bridge that maps a -frozen calibrated probability-of-default model through Mondrian conformal -intervals into robust portfolio constraints and an empirical funded-set audit. -On a 276,869-loan out-of-time Lending Club evaluation, the selected policy earns -`$184.8K` on a `$1M` budget while passing the declared eight-level alpha grid -($V(0.01)=0.035350$, $\Gamma_{\mathrm{CP}}=0.162616$, Markov cap `0.345084`, -zero violation). The same exact frontier exposes conservative and economic -endpoints, making the return-bound trade-off visible rather than implicit. -Frozen external credit replications preserve the predeclared global conformal -gates and produce positive robust LP objectives, so the result is not confined -to the Lending Club panel. Across the four frozen external applications, the -price of robustness forms a positive premium series ordered by panel default -rate. The contribution is a conformal-robust -credit-portfolio decision certificate with a -distribution-free Markov bound under weighted funded-set validity: it connects -real credit data, calibrated predictive models, robust funding decisions, and a -validation harness that rebuilds the prediction-to-decision chain from frozen -inputs while keeping the statistical guarantee boundary explicit. - -**Keywords:** conformal prediction; robust optimization; predict-then-optimize; -credit risk; portfolio optimization; reproducible data science. +Credit models matter only through the decisions they change. We study how +finite-sample predictive uncertainty can constrain a loan portfolio after a +probability-of-default (PD) model has been frozen. Conformal Robust +Predict-Then-Optimize (CRPTO) recomputes a 90% Mondrian conformal upper endpoint +exactly, forms the transparent decision score $q_i=(p_i+u_i)/2$, and places +$q_i$ in a portfolio-risk constraint while retaining point PD in the economic +objective. Nine round-number policies are ranked on November 2017 without +default, realized-return, miscoverage, or assumption-conditional columns; an +outcome-free December replay selects the same rule before its outcomes are +opened. That audit misses nominal funded-set coverage, making the statistical +boundary observable rather than rhetorical. The policy is then replayed on +276,869 out-of-time Lending Club loans. +It funds 308 loans and earns `$179,327.59` on a `$1M` budget, with weighted +default `0.039375`, weighted miscoverage `0.036875`, and conformal endpoint +budget `0.258051`. A matched point-PD allocation earns `$196,369.14` but has +weighted default `0.118400` and endpoint budget `0.921317`. Thus CRPTO pays +`8.678%` of realized return for a `7.9025` percentage-point default reduction; +the advantage reverses in some temporal slices, so we do not claim universal +dominance. The contribution is a small, auditable prediction-to-decision +guardrail with an exact replay, an inspectable selector, and explicit statistical +boundaries, rather than another credit-scoring leaderboard or a collection of +policy variants. + +**Keywords:** conformal prediction; predict-then-optimize; credit risk; +portfolio optimization; calibration; reproducible data science. # Introduction -Credit allocation is a predict-then-decide problem. A lender first estimates a -probability of default (PD), then chooses which loans to fund under a budget and -risk appetite. The modeling literature has become very good at the first step: -calibration, discrimination, and backtesting are now standard ingredients of -credit-risk model validation [@lessmann2015; @chen2024creditrisk]. The second step is -less settled. Once a calibrated PD enters an optimizer, uncertainty is often -treated as a reporting diagnostic rather than as a constraint that can change -the funded set. - -That separation is uncomfortable in auditable credit decisions. A portfolio -policy can have a reasonable average PD and still concentrate probability mass -in loans where the model is most uncertain. Conversely, a policy that is too -conservative can pass every risk check while destroying economic value. The -scientific question in this paper is therefore not whether one can build a -slightly better credit classifier. It is whether finite-sample predictive -uncertainty can be carried into a robust portfolio decision in a way that is -transparent enough for a reviewer to audit. This has practical stakes. In a -pre-registered randomized trial, conformal prediction sets -improved human decision making relative to fixed-size sets with the same -coverage [@cresswell2024]. CRPTO takes that committee-facing idea into a credit -portfolio setting, where the uncertainty summary must change a funding decision -or it is just another report. - -CRPTO answers this question with a post-hoc, reproducible pipeline. It starts -from a calibrated CatBoost PD model, constructs Mondrian conformal intervals -over PD-scale predictions, and maps the upper conformal endpoint into robust -portfolio constraints. The pipeline is modular by design: the predictive model, -conformal layer, optimization policy, and paper outputs each have separate -contracts. That separation lets the paper ask whether a frozen prediction -system can be converted into a defendable decision system without reopening -hyperparameter search whenever the manuscript or appendix changes. - -The empirical setting is the Lending Club retail-loan panel, with an -out-of-time evaluation set of 276,869 loans. The consolidated frontier contains -50,010 deduplicated semantic policies, of which 27,508 pass every declared -alpha level and exceed the return floor. From that declared finite frontier, -the selected policy is the highest-return point that passes the eight -alpha levels under the declared `0.345` Markov cap; it is neither a continuous -global optimum nor the economic endpoint. The selected point earns `$184.8K` -on a `$1M` budget and passes the exact empirical funded-set audit at -$\alpha = 0.01$. The headline result is not a single lucky allocation. It is a -reproducible bridge from calibrated probabilistic learning to robust, auditable -credit portfolio choice, with the return-bound frontier reported rather than -hidden behind one selected point. - -To address the natural "single dataset" concern without reopening the Lending Club -champion, we also freeze two external economic replications: Prosper final-status -marketplace loans and a Freddie/Mendeley single-family mortgage panel with -out-of-sample and out-of-time splits. These replications are not new champions; -they test whether the same PD-to-conformal-to-LP recipe remains economically -usable on different credit products. - -The paper makes five contributions. First, it gives a CRPTO construction for -credit portfolios: frozen calibrated PD, Mondrian conformal uncertainty, and -robust budgeted optimization as a post-hoc decision audit. Second, it proves a -distribution-free Markov bound under weighted funded-set validity (Theorem 1) -that splits realized portfolio loss into the conformal upper-endpoint budget -$B_u(\alpha) = \tau + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ and the weighted -miscoverage $V(\alpha)$, with supplement propositions showing that Markov is -optimal under the stated assumption (A.1) and locating the cluster structure -that would tighten it (A.2). Third, it locates that construction relative to -data-driven robust optimization, P2P lending portfolio models, conformal credit -scoring, and decision-focused learning. Fourth, it provides an evidence-backed -empirical study where every table and figure is generated from frozen outputs -rather than manually transcribed summaries. Fifth, it uses a three-level -evidence ladder: the Lending Club funded set carries the certificate, Prosper -and Freddie/Mendeley test recipe transfer as external economic replications, -and the tail, dependence, online, and robustness appendices stay diagnostic. -The key claim is narrow: CRPTO maps frozen calibrated PD models into a robust -funded set, reports the portfolio-level conformal premium -$\Gamma_{\mathrm{CP}}$, and verifies exact alpha-safe weighted miscoverage on the -promoted Lending Club portfolio and its surrounding finite-grid frontier. The -same conformal and LP gates remain viable on two additional credit datasets, -while the paper keeps the boundary between paper revision and new model-search -evidence visible. - -Read as data science for decisions, the paper's four components are explicit. -The data component is a static Lending Club OOT panel, with Prosper and -Freddie/Mendeley replications used only as frozen external stress tests. The -model component is the CRPTO bridge from calibrated PD to Mondrian conformal -uncertainty to robust LP. The decision component is the funded set under a -budget and risk cap. The implication is a reproducible audit surface for model -risk committees: the paper reports not only return, but also the conformal -premium, exact funded-set miscoverage, region stability, and the boundary beyond -which a new protocol would be required. - -![CRPTO's central move is operational rather than decorative: frozen calibrated PD models become Mondrian conformal intervals, the upper-risk signal enters a robust portfolio optimizer, and the selected funding policy is audited against exact funded-set controls.](../reports/crpto/figures/crpto_fig1_journal_pipeline.png){#fig-crpto-pipeline width="94%" fig-alt="Four-stage CRPTO pipeline from frozen calibrated PD to Mondrian conformal intervals, robust portfolio optimization, and selected auditable policy."} +Credit allocation is a contextual optimization problem. A lender estimates a +PD and then chooses which loans to fund under capital, concentration, and risk +constraints [@sadana2025contextual]. Credit-scoring research has made the first +step increasingly reliable through discrimination benchmarks, probability +calibration, and cost-aware evaluation [@lessmann2015; @chen2024creditrisk; +@yang2025costaware]. Yet a calibrated probability does not specify how much +model uncertainty a portfolio should bear. A point-PD optimizer can concentrate +capital in loans that look attractive precisely where the predictive model is +least certain. + +Conformal prediction offers finite-sample coverage language under explicit +exchangeability conditions [@vovk2005; @angelopoulos2023], while robust +optimization makes uncertainty operational through feasibility sets +[@bertsimas2004; @goldfarb2003robustportfolio]. Joining the two is attractive, +but an applied paper still has to answer three questions. Which conformal level +is informative rather than nearly vacuous? How is a policy selected without +using the outcomes on which it is later reported? And what economic value is +lost when conformal uncertainty actually changes the funded set? + +CRPTO answers those questions with one deliberately simple policy. A frozen, +calibrated CatBoost model produces $p_i$. An exactly replayed 90% Mondrian +recipe produces upper endpoint $u_i$. The portfolio uses their midpoint, + +$$ +q_i = p_i + 0.5(u_i-p_i) = \frac{p_i+u_i}{2}, +$$ + +inside the risk constraint. The economic objective remains point-PD expected +net return. This separation is important: $p_i$ prices expected loss, while +$q_i$ limits the amount of uncertainty the funded portfolio may carry. It also +removes the capped, tail-focused, and uncertainty-penalty branches that made an +earlier research frontier difficult to explain and easy to misread. + +The empirical design uses a temporal Lending Club panel. The conformal recipe +is fit inside the calibration period, November 2017 ranks a declared +$3\times3$ grid of round-number risk tolerances and conformal weights, and +December replays the outcome-free selector before auditing the already-fixed +decision. The ranking artifact contains no defaults, realized returns, +miscoverage, or assumption-conditional statistics. The fixed rule is then +evaluated on loans originated from January 2018 through September 2020. Earlier +project development did inspect this static OOT corpus, so we describe the +final run as a transparent retrospective lockbox replay, not as a pristine +prospective or preregistered trial. + +The paper makes three contributions. First, it gives an auditable +prediction-to-decision construction in which the economic objective and the +conformal guardrail have separate, inspectable roles. Second, it replaces +approximate cross-alpha scaling with an exact replay, a deterministic endpoint +screen, and a temporally separated selector audit. Third, it reports the price +and limits of that guardrail against matched point-PD and more-conservative +comparators, including a pre-OOT funded-set coverage miss and temporal slices +where CRPTO wins and slices where it does not. The novelty is the closed, +inspectable decision protocol for a frozen credit model, not a claim that +conformal prediction, robust optimization, or credit scoring is individually +new. + +![CRPTO carries a frozen calibrated PD through an exact conformal replay, a simple portfolio guardrail, and a funded-set audit.](../reports/crpto/figures/crpto_fig1_journal_pipeline.png){#fig-crpto-pipeline width="92%" fig-alt="Four-stage CRPTO pipeline from calibrated PD to conformal intervals, portfolio allocation, and funded-set audit."} # Related Work -CRPTO builds on conformal prediction, especially split conformal methods and -their risk-control extensions [@vovk2005; @angelopoulos2023; -@angelopoulos2024risk]. The relevant property is not that conformal intervals -are the narrowest possible uncertainty summaries. It is that they provide -distribution-free coverage language under explicit exchangeability assumptions, -and that this language can be audited without trusting a fully parametric -posterior. Mondrian and group-conditional variants are especially natural in -credit because risk grades are already used as business and governance -partitions [@bostrom2021; @gibbs2024]. The paper keeps the stronger localized, -weighted, and conditional claims separate because recent impossibility and -weighted-coverage results show that those guarantees require extra structure -[@barber2021limits; @barber2023beyond; @jonkers2024wcps]. - -The second foundation is robust optimization. Classical robust optimization -frames uncertainty as a set against which a decision must remain feasible, with -the price of robustness made visible as a design trade-off [@bertsimas2004]. -Robust portfolio selection makes that trade-off operational for allocation -under parameter uncertainty [@goldfarb2003robustportfolio], whereas -distributionally robust optimization broadens the uncertainty object toward -moment or ambiguity sets [@delage2010dro]. Data-driven robust and -prescriptive-analytics work then connects predictive models to downstream -decisions while keeping the uncertainty-to-action contract explicit -[@bertsimas2018datadriven; @bertsimas2020prescriptive]. Recent work connects -conformal prediction and robust optimization more directly by using conformal -uncertainty sets in downstream decisions [@johnstone2021; @patel2024; -@hu2026crc]. That line certifies the uncertainty set *before* the decision: -coverage of the conformal region is the guarantee, and the downstream decision -inherits it. CRPTO follows this line but audits the other side of the decision -as well: after the optimizer selects a funded set, the realized weighted -miscoverage $V(\alpha)$ and the deterministic budget identity of Theorem 1 are -evaluated exactly on that funded set, so the certificate concerns the -allocation actually purchased rather than the input set alone. Its empirical -emphasis also differs: the uncertainty set is not an abstract benchmark -instance, but a credit-risk interval model with auditable lineage, paper tables, -and model-risk documentation. - -The third foundation is predict-then-optimize and decision-focused learning. -IJDS work on causal decision making sharpens the same warning: once an estimate -feeds an action, the relevant target can become the assignment rule rather than -only the intermediate effect-size estimate [@fernandezloria2022causaldecision]. -SPO+ and modern decision-focused learning -ask models to respect the loss surface induced by the downstream decision -[@elmachtoub2022; @donti2017; @mandi2024]. CRPTO is more -conservative. It does not retrain the PD model end-to-end through the optimizer. -Instead, it asks what can be achieved when a calibrated predictive system is -already frozen and the decision layer must remain explainable to credit-risk -reviewers. Robust losses for decision-focused learning [@schutte2024robust] -share this protective intent, but operate at training time rather than as a -post-hoc auditable constraint. - -The fourth foundation is machine learning and optimization for credit -decisions. Credit-scoring benchmarks define the performance frontier on retail -panels [@lessmann2015; @ayari2026; @xia2017]. Recent IJDS credit-risk work -shows how richer data structures such as firm graphs can improve rating -prediction [@das2023creditgraph], and cost-aware calibration work makes explicit -why probability quality matters when predictions feed asymmetric downstream -decisions [@yang2025costaware]. Work on fintech lending and consumer-credit -allocation studies platform structure, credit invisibility, measurement noise, -and scorecard equity [@jagtiani2019altdata; @albanesi2024credit; -@khandani2010consumer; @fuster2022predictably]. -In the P2P/Lending Club decision neighborhood, prior work studies -instance-based investment support, P2P portfolio selection, profit scoring, -robust credit portfolio optimization, and multi-objective AI/OR funding -policies [@guo2016p2p; @zhao2016p2pportfolio; @serrano2016profitscoring; -@chi2019p2p; @babaei2020p2p; @aior2025lendingclub]. Recent ordinal conformal -credit-scoring work also means that the safe claim is not "no conformal -prediction in credit" [@kawasumi2026ordinal]. CRPTO does not compete on raw -ranking against this literature; its champion AUC is mid-range. -The contribution is the auditable bridge from a calibrated, frozen PD model to a -conformal robust portfolio decision, not another point on the credit-scoring -leaderboard. +CRPTO sits at the intersection of conformal prediction, robust optimization, +and decision-focused learning. Split conformal methods provide marginal +coverage without a parametric posterior, while Mondrian variants condition on +declared partitions [@vovk2005; @bostrom2021]. Conditional and weighted +extensions require additional structure, and exact conditional coverage is +generally unavailable without restrictive assumptions [@barber2021limits; +@barber2023beyond; @jonkers2024wcps]. We therefore distinguish population or +partition coverage from coverage after a portfolio has adaptively reweighted +the loans. + +Data-driven robust and contextual optimization translate predictive +uncertainty into decisions [@bertsimas2018datadriven; +@bertsimas2020prescriptive; @sadana2025contextual]. Recent work uses conformal +sets directly in robust optimization [@johnstone2021; @patel2024; +@sun2024ptc; @hu2026crc]. CRPTO is an applied complement: it retains a frozen +credit PD model, exposes the exact funded rows and uncertainty premium, and +compares the resulting allocation with a matched point-PD decision. + +Decision-calibrated prediction sets and inverse conformal risk control go +further by calibrating downstream violation or regret and, after choosing a +robustness level, using a separate split to restore risk-estimation validity +[@zhou2026creme; @stratigakos2026decision_calibrated_sets]. CRPTO does not +import that guarantee into a batch portfolio where the optimizer chooses the +funded weights. Instead, it separates a deterministic endpoint-budget screen +from an independent post-selection audit and reports when selected-set +miscoverage misses its nominal target. + +Decision-focused learning and SPO+ train predictions against downstream regret +[@donti2017; @elmachtoub2022; @mandi2024]. That is a different institutional +choice. CRPTO asks what can be done after a calibrated model already exists and +must remain unchanged for governance. A synthetic SPO+ comparison is retained +in the supplement because it confirms the expected distinction: decision- +focused training improves its own regret objective, whereas CRPTO produces an +auditable uncertainty-constrained funded set. + +Credit-allocation research already covers profit scoring, P2P investment +recommendation, robust loan portfolios, rejection, and multiobjective risk +[@guo2016p2p; @zhao2016p2pportfolio; @serrano2016profitscoring; @chi2019p2p; +@babaei2020p2p; @xu2025profit_uncertainty_credit; +@xu2024profit_risk_credit]. Conformal credit scoring also means that the safe +claim is not "first use of conformal prediction in credit" +[@kawasumi2026ordinal]. The remaining gap is a file-backed protocol that shows +exactly how a conformal endpoint changes a budgeted credit decision and what +that change costs. + +| Literature family | Existing contribution | CRPTO boundary | +|---|---|---| +| Credit scoring and calibration | Accurate, calibrated PD and cost-aware prediction. | Treats PD as an input contract; does not claim AUC leadership. | +| P2P and robust credit portfolios | Economic loan selection under risk and uncertainty. | Uses an exact conformal endpoint as a simple portfolio guardrail. | +| Conformal robust optimization | Coverage-backed uncertainty sets for downstream decisions. | Adds a frozen credit stack, funded-set audit, and matched economic comparator. | +| Decision-focused learning | Training-time reduction of decision regret. | Keeps the predictive model frozen and emphasizes post-hoc auditability. | +| Decision-risk calibration and valid selection | Calibrates decision losses or corrects validity after choosing among sets [@zhou2026creme; @hegazy2025valid_selection_conformal_sets]. | Uses a deterministic selector plus an independent diagnostic audit; formal selected-set validity is not asserted. | + +: Closest-work boundary for the submitted CRPTO claim. + +# Data and Evaluation Design -Finally, recent work on conformal model selection for robust optimization, -multi-distribution conformal validity, online conformal portfolio methods, -end-to-end conformal risk training, robust conformal decision certificates, and -conformal satisficing motivates the journal-strengthening package -[@bao2025croms; @yang2026multidistribution; @liu2026portfolio; -@zhou2025credo; @zhou2026creme; @zhao2025robust]. We use those ideas where they -can be evaluated from the frozen CRPTO evidence: -OCE/CVaR [@rockafellar2000cvar; @bental2007oce] appears as a tail-risk -diagnostic, robust satisficing appears as committee-style margin evidence, and -SPO+ motivates the regret-auditability frontier. The external Prosper and -Freddie/Mendeley runs are a separate, frozen replication protocol rather than a -new method-changing search. The remaining method-changing variants--optimized -OCE/CVaR objectives, online protocols, causal layers, and hybrid -decision-focused training--remain future work rather than hidden acceptance -criteria. - -## Closest Work Boundary - -Table 1 states the novelty boundary directly. CRPTO is not first in any broad -individual family; its claim is the combination of calibrated PD, conformal -uncertainty, robust credit-portfolio optimization, exact funded-set validation, -and reproducible governance. The row labels aggregate the closest -families discussed above: P2P/Lending Club OR [@guo2016p2p; -@zhao2016p2pportfolio; @serrano2016profitscoring; @chi2019p2p; @babaei2020p2p; -@aior2025lendingclub], conformal credit scoring [@kawasumi2026ordinal], -conformal robust optimization [@johnstone2021; @patel2024; @hu2026crc], -decision-focused learning [@elmachtoub2022; @mandi2024; @schutte2024robust], -and conformal finance portfolios [@noguer2024portfolio; @kato2025; -@liu2026portfolio]. - -| Neighboring literature | What it already contributes | What CRPTO adds | Why not enough for auditable credit decisions | -|---|---|---|---| -| P2P/Lending Club OR | Credit investment recommendation and robust/multi-objective funding models. | Conformal PD uncertainty as the uncertainty set plus exact funded-set validation. | Leaves a gap between prediction uncertainty and a post-allocation certificate that a credit committee can inspect. | -| Conformal credit scoring | Conformal intervals for ordinal credit scores. | A downstream robust portfolio decision, not only score uncertainty. | Stops at score uncertainty; it does not audit a budgeted funded set or economic policy. | -| Conformal robust optimization | Conformal sets used in downstream robust decisions. | A frozen Lending Club credit-risk model stack with paper-facing audit trail. | Establishes the decision logic, but not the credit-specific PD lineage and funded-set certificate. | -| Decision-focused learning | Training losses aligned with downstream regret. | A post-hoc governance path for institutions with existing calibrated PD models. | Improves training-time regret, but does not certify risk controls after a frozen production-style PD model. | -| Conformal finance portfolios | Portfolio use cases for financial markets. | Retail credit payoffs, default risk, funded-set checks, and model-risk documentation. | Market-return portfolios have different payoffs and governance obligations than binary default-driven credit funding. | - -: Closest work boundary for CRPTO. - -The novelty is the closed loop, not any single row of the table: a frozen -calibrated credit model becomes conformal upper endpoints, those endpoints -enter a robust funded-set decision, and the purchased allocation is audited -after the decision with file-backed governance. None of the neighboring lines -jointly supplies that frozen-model-to-funded-set certificate for a credit -portfolio. +The data are Lending Club retail loans originated from 2007 through 2020. The +feature contract contains only information available at origination. The model +and evaluation pipeline use temporal rather than random splits. + +| Split | Period | Loans | Role | +|---|---|---:|---| +| Train | Jun 2007--Mar 2017 | `1,346,311` | Fit the PD model and calibrator. | +| Conformal fit | Mar--Oct 2017 | `142,550` | Estimate the frozen conformal recipe. | +| Policy selection | Nov 2017 | `14,943` | Rank the nine outcome-free policies. | +| Calibration audit | Dec 2017 | `20,695` | Replay the selector, then open outcomes. | +| OOT evaluation | Jan 2018--Sep 2020 | `276,869` | Freeze-then-evaluate portfolio decisions. | + +: Temporal Lending Club design. + +The frozen conformal recipe uses the most recent 75% of the original +calibration pool (`178,188` rows). Within that subset, `142,550` rows estimate +conformal quantiles, `14,943` November rows select the policy, and `20,695` +December rows form a pre-OOT audit. The conformal recipe uses calibration +labels, as conformal prediction requires. The loader stores holdout outcomes +separately from the 12-column policy frame, which contains identifiers, +origination context, PD endpoints, amounts, and rates. Ranking cannot access +outcomes, realized returns, miscoverage, or the assumption-conditional Markov +quantity. December first reruns that same outcome-free selector and only then +joins outcomes for the decision audit. + +The final OOT panel covers several regimes, including the 2020 disruption. We +report the full panel and five temporal slices (`2018H1`, `2018H2`, `2019H1`, +`2019H2`, and `2020+`). Each slice solves the same fixed policy on a fresh `$1M` +budget, so slice returns are comparable stress evaluations rather than +components that sum to the full-panel return. # Method -## Calibrated PD Layer - -The predictive layer estimates a one-period default probability for each loan. -The champion model is a CatBoost classifier trained on the frozen feature -contract and calibrated before it is exposed to conformal and optimization -layers. The paper reports discrimination and probability quality together: -the PD layer reaches AUC `0.7139`, Brier score `0.1544`, and expected -calibration error approximately `0.0070` on the paper-facing evaluation -summary. These numbers matter because the optimizer consumes probabilities, not -rankings alone. - -Calibration is treated as a contract. The downstream layers do not receive a -free-form classifier; they receive a calibrated PD vector, feature metadata, -and a model contract that fixes feature order and categorical handling. This -prevents a common reproducibility failure in applied predict-then-optimize -studies: a table can change because a preprocessing file changed, even -though the optimization code did not change. - -## Mondrian Conformal Layer - -For each loan `i`, let `p_hat_i` denote the calibrated PD. The conformal layer -forms prediction intervals on the PD scale and records the upper endpoint -`u_i(alpha)`. Operationally, CRPTO evaluates Mondrian partitions rather than a -single global interval. The selected uncertainty layer uses a -score-decile-based Mondrian partition selected by out-of-time interval quality, -while grade-based partitions remain the natural governance baseline. The -score-decile choice keeps each cell well populated even at the tight -$\alpha = 0.01$, where a per-cell $99\%$ upper endpoint needs on the order of -$100$ calibration points; finer grade-period partitions are sparser, and the -supplement flags where small cells weaken coverage (notably the external Freddie -panel). - -The resulting conformal summary is more than a scalar coverage -number. The paper-facing metrics include 90% coverage `0.9297`, 95% coverage -`0.9664`, average 90% interval width `0.7842`, minimum group 90% coverage -`0.9190`, and 90% Winkler score `1.1107` for the promoted conformal winner. -Because the outcome is binary and the intervals live on the clipped PD scale, -raw width is not a standalone utility claim. Its decision role is relative: -upper endpoints rank loans by protected downside risk, the funded-set audit -shows mean upper endpoints rising from `0.12529` in A-B to `0.52587` in E-G, and -the promotion gate uses Winkler score, funded-set miscoverage, and -$\Gamma_{\mathrm{CP}}$ rather than treating narrowness as the objective. -The full gate also scores material coverage, group coverage, interval width, and -alert rate, but not exact-nominal-coverage backtests: conformal intervals -over-cover by design, so testing for exact nominal coverage would penalize the -safety margin the method is meant to provide on a large out-of-time sample. - -## Robust Portfolio Layer - -The decision variable $x_i$ is the allocation fraction for each eligible loan; -$x_i a_i$ is the funded exposure. The optimizer maximizes expected net -economic return under a `$1M` budget and policy constraints that cap portfolio -risk after replacing point PD estimates with conformal upper endpoints. The -selected body point has -`risk_tolerance = 0.1715`, `policy_mode = capped_blended_uncertainty`, policy -parameter `gamma = 0.5475`, and `uncertainty_aversion = 0.05`. - -Schematically, the robust decision layer solves a budgeted allocation problem -of the form +## Calibrated PD and Exact Conformal Replay + +Let $p_i\in[0,1]$ be the calibrated PD for loan $i$. The predictive layer is a +frozen CatBoost classifier with AUC `0.7139`, Brier score `0.1544`, and expected +calibration error about `0.0070` on the paper-facing evaluation. These values +are not presented as a leaderboard result; probability quality matters because +the downstream objective consumes PD directly. + +The conformal recipe partitions calibrated scores into five score-quantile +Mondrian cells. For calibration loan $j$, it computes a scaled residual + +$$ +s_j = \frac{|Y_j-p_j|}{\sqrt{p_j(1-p_j)}}, +$$ + +with numerical clipping near zero. Each cell uses the finite-sample `higher` +quantile at the frozen conservative level `0.095` for the target +$\alpha=0.10$. Recorded group and temporal widening factors are then applied +without narrowing. The upper endpoint is +$u_i=\min\{1,p_i+\widehat q_{g(i)}\sqrt{p_i(1-p_i)}\}$ after those frozen +adjustments. + +The replay implementation reconstructs this recipe from its result payload. At +the reference 90% level, it reproduces the stored point, lower, and upper +vectors with maximum absolute error below `6.67e-16`. This matters because an +earlier exploratory analysis scaled 90% row radii using average widths from a +different conformal family. Those cross-alpha values are retained only as +historical provenance and are not used here. + +## One Portfolio Policy + +Let $a_i$ be loan amount, $x_i\in[0,1]$ the funded fraction, $r_i$ the coupon, +$L=0.45$ loss given default, $B=1{,}000{,}000$, and $\tau$ the portfolio risk +tolerance. CRPTO solves $$ \begin{aligned} -\max_x\quad & \sum_i x_i a_i \left(c_i - \tilde p_i(\alpha,\gamma)\,L\right) \\ -\text{s.t.}\quad & \sum_i x_i a_i \le B,\\ -& \sum_i x_i a_i \tilde p_i(\alpha,\gamma) - \le \tau \sum_i x_i a_i,\\ -& 0 \le x_i \le \bar x_i, +\max_x\quad & \sum_i x_i a_i(r_i-p_iL) \\ +\text{s.t.}\quad +& \sum_i x_i a_i \le B, \\ +& \sum_i x_i a_i q_i \le \tau\sum_i x_i a_i, \\ +& 0\le x_i\le \bar x_i, \end{aligned} $$ -where `a_i` is exposure, `c_i` is the loan coupon (interest rate), `L` is the -loss-given-default (`L = 0.45` in the frozen evaluation), $\tau$ is the -risk-tolerance cap, and +with the existing concentration and eligibility constraints. The selected +policy uses $\tau=0.17$ and $q_i=(p_i+u_i)/2$. The objective is intentionally +based on $p_i$, not $q_i$: expected economics and uncertainty feasibility are +separate contracts. The matched point-PD baseline changes only the risk score +to $q_i=p_i$ while holding candidates, budget, concentration, LGD, solver, and +risk tolerance fixed. + +For funded-exposure weights +$w_i=x_ia_i/\sum_jx_ja_j$, define $$ -\tilde p_i(\alpha,\gamma) -= \hat p_i + \gamma\left(u_i(\alpha)-\hat p_i\right) +\begin{aligned} +\Gamma_{\mathrm{CP}} &= \sum_iw_i(u_i-p_i),\\ +\Gamma_{\mathrm{int}} &= \sum_iw_i(q_i-p_i),\\ +\Gamma_{\mathrm{res}} &= \sum_iw_i(u_i-q_i),\\ +B_u &= \sum_iw_i u_i. +\end{aligned} $$ -on the PD scale, clipped to the feasible probability range. The objective is the *expected* -net return `c_i - p_tilde_i * L`; the headline realized return is the -post-hoc accounting of the same funded set on observed defaults -(a funded loan earns `c_i * a_i` if it survives and loses `L * a_i` if it -defaults). Separating the optimized expectation from the realized accounting is -deliberate: the optimizer never sees outcomes, so the realized figure is an -out-of-sample audit of the policy, not the objective it maximized. Additional -operational filters and caps live in the frozen policy configuration; the body -displays the core statistical-to-decision contract because that is the reusable -CRPTO pattern. - -Two quantities are kept separate throughout the paper. The lowercase $\gamma$ -is a policy parameter controlling how the optimizer blends uncertainty in the -portfolio rule. $\Gamma_{\mathrm{CP}}$, by contrast, is a portfolio-level conformal metric -computed after the funded set is chosen: it is the allocation-weighted gap -between the conformal upper endpoint and calibrated PD, with clipping at one -on the PD scale. In committee language, $\Gamma_{\mathrm{CP}}$ is the realized conformal -robustness premium paid by the funded set. The selected body point has -$\Gamma_{\mathrm{CP}}(\alpha = 0.01) = 0.162616$, weighted miscoverage -$V(\alpha = 0.01) = 0.035350$, and zero exact violation at $\alpha = 0.01$. This -distinction is small typographically but central for auditability. - -# Theory - -The theoretical role of conformal prediction in CRPTO is modest and explicit. -For a fixed allocation evaluated on exchangeable calibration/test data, -conformal coverage controls the expected rate at which outcomes fall outside -the constructed uncertainty intervals. When funded-set weights are -non-negative and normalized, this yields a weighted miscoverage quantity -$V(\alpha)$ that can be monitored after the decision. Two statements are kept -separate. The deterministic portfolio identity (Theorem 1(i)) holds for any -allocation and needs no distributional assumption; it is the accounting the -exact certificate verifies. The probabilistic statement (Theorem 1(ii)) is a -Markov argument that requires weighted funded-set validity, -$E[V(\alpha)] \leq \alpha$, stated as Assumption 1. This is a modeling -assumption, not a property the single frozen draw establishes: on the selected -funded set the realized weighted miscoverage is $V(0.01) = 0.035350$, *above* -the nominal $\alpha = 0.01$, which is the expected price of evaluating an -adaptively selected subportfolio rather than a fresh population. What the paper -certifies is therefore the exact accounting together with the safety level -$V \leq \sqrt{\alpha}$ that Markov delivers, not a claim that the funded set -attains nominal $\alpha$-coverage or that post-selection evaluation creates a -stronger conformal guarantee. - -| Claim component | CRPTO evidence | Boundary | -|---|---|---| -| Deterministic portfolio identity | Exact funded-set audit computes $V$, $\Gamma_{\mathrm{CP}}$, and violation after allocation. | Does not require a new statistical guarantee. | -| Split/Mondrian validity | OOT coverage, minimum group coverage, and temporal diagnostics. | Not exact conditional coverage for every borrower profile. | -| Weighted funded-set validity | Exact alpha-safe certificate plus A23 weighted/multi-distribution stress evidence. | Assumption for the theorem; empirical audit after frozen selection. | -| Post-selection robustness | Nested search chain plus a declared finite-grid frontier with 50,010 deduplicated semantic policies and 27,508 all-alpha above-floor policies. | Strong audit evidence, not a prospective live-selection guarantee or a continuous-region theorem. | +For the midpoint policy, +$\Gamma_{\mathrm{int}}=\Gamma_{\mathrm{res}}=\Gamma_{\mathrm{CP}}/2$. +This identity is one reason to prefer the midpoint over nonlinear caps or tail +rules: every quantity has a direct interpretation. -: Assumption-to-evidence map for the CRPTO bound. +## Calibration-Only Final Selector -The figure below is the paper's main guardrail against overclaiming: it -separates what is deterministic, what is assumed, and what is empirically -certified after the frozen selection. +The candidate set crosses +$\tau\in\{0.15,0.17,0.19\}$ with +$\gamma\in\{0.25,0.50,0.75\}$ in +$q_i=p_i+\gamma(u_i-p_i)$. A candidate is eligible when the solver is optimal, +at least 99.9% of the budget is allocated, the effective-PD cap holds, and -![The bound claim stack separates deterministic accounting, the weighted-validity assumption, and the frozen exact certificate.](../reports/crpto/figures/crpto_fig20_bound_claim_layers.png){#fig-bound-claim-stack width="94%" fig-alt="Four-block bound claim stack separating conformal endpoint, deterministic identity, weighted validity assumption, and exact frozen certificate."} +$$ +B_u\le 0.28 +$$ -Dependence is handled conservatively rather than assumed away. The main bound -does not require loan-level independence. Temporal structure is addressed by -the out-of-time design and backtests; any sharper concentration argument is -kept in the supplement, where the extra independence structure is stated -explicitly. +on the November block. Among eligible candidates, the rule maximizes expected +point-PD objective. Five of nine candidates pass, and the selected candidate is +$\tau=0.17,\gamma=0.50$. The endpoint cap is deterministic; it does not use +the weighted-validity assumption introduced below. The selected row remains +optimal for every cap in $[0.259036,0.290491)$, so the round `0.28` value has +positive margin to both policy-change boundaries. An outcome-free replay on +December independently selects the same candidate. -The compact validity ladder below fixes that boundary. CRPTO uses the first two -levels as evidence, states weighted funded-set validity as the theorem's -portfolio-level assumption, reports multi-distribution checks as diagnostics, -and leaves online/live control for a new protocol. +| Candidate | $\tau$ | $\gamma$ | Nov. expected objective | Nov. $B_u$ | Dec. $B_u$ | Status | +|---|---:|---:|---:|---:|---:|---| +| Higher-return, low guardrail | `0.17` | `0.25` | `$109,885.56` | `0.375105` | `0.396751` | Ineligible | +| Selected midpoint | `0.17` | `0.50` | `$99,387.12` | `0.259036` | `0.262082` | Selected twice | +| More-conservative blend | `0.17` | `0.75` | `$93,760.13` | `0.202259` | `0.203504` | Eligible | +| Higher risk tolerance | `0.19` | `0.50` | `$102,671.87` | `0.290491` | `0.294861` | Ineligible | -| Validity level | What it supports | CRPTO status | -|---|---|---| -| Marginal split conformal | Population-level coverage under exchangeability. | Core interval guarantee. | -| Mondrian/group conformal | Coverage within declared partitions. | Used for score-decile and grade audits. | -| Weighted/localized coverage | Coverage under weights, local neighborhoods, or selected groups. | Explicit theorem assumption plus A23 diagnostic evidence. | -| Multi-distribution coverage | Robustness across multiple source distributions. | Read-only stress evidence, not recalibration. | -| Online/adaptive coverage | Sequential alpha updates under live drift. | A24 replay only; not a live deployment claim. | - -: Coverage-validity ladder used in the paper claims. - -The bound is therefore read as three linked objects: a deterministic -accounting identity, an explicitly stated statistical assumption, and the -frozen empirical certificate that verifies the promoted decision exactly on -the 276,869-loan OOT evaluation. We now state the first two formally. - -**Setup.** Fix the promoted allocation $x$, chosen without access to OOT -labels, with exposures $a_i > 0$ and funded-set weights -$w_i = x_i a_i / \sum_j x_j a_j$, so that $w_i \geq 0$ and $\sum_i w_i = 1$. -For each funded loan let $Y_i \in [0, 1]$ be the realized outcome on the PD -scale, $u_i(\alpha) \in [0, 1]$ its upper conformal endpoint, and -$Z_i(\alpha) = \mathbf{1}\{Y_i > u_i(\alpha)\}$ the miscoverage indicator. -The weighted funded-set miscoverage is -$V(\alpha) = \sum_i w_i Z_i(\alpha)$. Probabilities and expectations below are -taken over the exchangeable calibration/test draw, conditional on the frozen -recipe, declared partitions, and allocation rule. - -**Assumption 1 (weighted funded-set validity).** -$E[V(\alpha)] \leq \alpha$ under the funded-set weights $w$ for that draw. This -means the expected weighted miss rate on the funded dollars is no larger than -the nominal conformal level. It is the explicit price of evaluating a selected -portfolio rather than a single population, and it does not follow from marginal -split conformal. The -funded-set weights $w_i \propto x_i a_i$ are chosen by the optimizer and depend -on the conformal endpoints $u_i(\alpha)$, so they are not measurable with -respect to the Mondrian partition and inherit no per-cell coverage guarantee. -The assumption is therefore stated, audited empirically after the frozen -selection, and never silently upgraded to a guarantee -- and the audit does not -find it slack: the realized $V(0.01) = 0.035350$ exceeds $\alpha = 0.01$, so the -operative safety level is the weaker $V \leq \sqrt{\alpha}$. - -**Theorem 1 (distribution-free Markov bound under weighted funded-set validity).** -Let $B_u(\alpha) = \sum_i w_i u_i(\alpha)$ be the weighted conformal -upper-endpoint budget of the funded set. Then: - -(i) *(deterministic)* $\;\sum_i w_i Y_i \leq B_u(\alpha) + V(\alpha)$ always; - -(ii) *(statistical)* under Assumption 1, for every $t > 0$, -$P(V(\alpha) \geq t) \leq \alpha / t$, and in particular +: Temporally separated selector examples. A36 reports all nine candidates on +both months. + +## Accounting and Statistical Boundary + +Let $Z_i=\mathbf 1\{Y_i>u_i\}$ and +$V=\sum_iw_iZ_i$. Because $Y_i\le u_i+Z_i$ for every funded loan, $$ -P\!\left(\sum_i w_i Y_i \;\geq\; B_u(\alpha) + \sqrt{\alpha}\right) - \;\leq\; \sqrt{\alpha}. +\sum_iw_iY_i\le B_u+V $$ -*Proof sketch.* Since $Y_i \leq u_i(\alpha) + Z_i(\alpha)$ for every loan, -part (i) follows by taking the $w$-weighted sum; it is portfolio accounting and -needs no probability. Part (ii) is Markov's inequality applied to the -nonnegative variable $V(\alpha)$ with $E[V(\alpha)] \leq \alpha$ [@ghosh2002], -combined with (i). The full proof is in Online Supplement Appendix A. $\square$ - -**The optimizer's cap versus the endpoint budget.** The robust layer -does not constrain $B_u(\alpha)$ directly; it caps the $\gamma$-blended PD, -$\sum_i w_i \tilde p_i(\alpha,\gamma) \leq \tau$, with -$\tilde p_i = \hat p_i + \gamma(u_i(\alpha) - \hat p_i)$ and $\gamma \in [0,1]$. -Because $\Gamma_{\mathrm{CP}}(\alpha) = \sum_i w_i(u_i(\alpha) - \hat p_i)$, the -endpoint budget decomposes exactly as +holds deterministically. It is an exact funded-set accounting statement, not a +coverage theorem. + +If one additionally assumes weighted funded-set validity, +$\mathbb E[V]\le\alpha$, Markov's inequality gives + $$ -B_u(\alpha) = \sum_i w_i \tilde p_i(\alpha,\gamma) + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha) - \;\leq\; \tau + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha), +\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt{\alpha}\right) +\le\sqrt{\alpha}. $$ -with equality when the risk cap binds. The term -$(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ is the conformal robustness premium the -optimizer leaves un-internalized at $\gamma < 1$. For the selected body point -($\tau = 0.1715$, $\gamma = 0.5475$, binding cap), -$B_u(0.01) \leq 0.1715 + 0.4525\,(0.162616) = 0.245084$, so the deterministic -accounting bound reads -$\sum_i w_i Y_i \leq 0.245084 + V(0.01) = 0.280434$. The exact audit reports -zero deterministic violation; the conservative Markov cap for the same point is -`0.345084`. - -**Remark 1 (why $t = \sqrt{\alpha}$, and why Markov).** -The choice $t = \sqrt{\alpha}$ is made for interpretability, not optimality: -it gives the clean reading "miscoverage exceeds $\sqrt{\alpha}$ with -probability at most $\sqrt{\alpha}$" (for $\alpha = 0.01$, a $0.10$ excess -with probability at most $0.10$). Markov is deliberately the weakest -defensible argument: it uses only the first moment. Supplement -Propositions A.1--A.2 separate the boundary: under Assumption 1 alone the -best second-moment (Cantelli) threshold is *worse* than Markov, while explicit -cross-cluster structure is the extra condition under which Hoeffding-style -tightening becomes available [@hoeffding1963; @boucheron2013concentration]. We -keep those tightenings in the online supplement (A21) rather than in the -body, because the contribution here is the auditable decision construction, -not the sharpest possible tail bound. The exact certificate in this paper is -the empirical audit of the frozen selected policy, not a stronger -post-selection conformal theorem. - -The theorem and the two supplement propositions should be read as one small -triptych. Theorem 1 gives the paper's guarantee once weighted funded-set -validity is accepted. Proposition A.1 shows that, without additional structure, -Markov is not a placeholder for a missing second-moment bound; it is the sharp -first-moment statement. Proposition A.2 then asks what extra structure would -buy a tighter threshold. In this temporal credit panel the defensible version is -cross-period or period-grade independence after the frozen recipe and allocation -are fixed: within a period, grade, or period-grade cell, defaults and interval -misses may remain dependent. The observed funded set is too exposure -concentrated for that cluster argument to tighten the headline bound, which is -why the body keeps Markov and the supplement reports the cluster calculation as -a sensitivity check. - -# Experimental Design - -The empirical study uses Lending Club retail-loan data covering originations -from 2007 through 2020. The raw panel is cleaned into a static feature store and -split temporally with a January 2018 cutoff, so that calibration and evaluation -use only loans originated after the training window. The calibration block plays -the dual role required by split conformal: it is held out from training and used -exclusively to fit the conformal quantiles. The final evaluation set contains -276,869 loans, large enough to stress both probability calibration and -decision-level robustness. -| Split | Period | Loans | Role | -|---|---|---:|---| -| Train | Jun 2007 -- Mar 2017 | `1,346,311` | Fit PD model and monotonic constraints. | -| Calibration | Mar 2017 -- Dec 2017 | `237,584` | Fit Mondrian conformal quantiles (held out from training). | -| Test (OOT) | Jan 2018 -- Sep 2020 | `276,869` | Evaluate coverage, run the portfolio decision, and audit the exact funded set. | - -: Temporal out-of-time split. The January 2018 cutoff keeps the test window -(including the 2020 COVID regime) strictly after training and calibration, so no -future information leaks into the funded-set decision. The displayed periods are -monthly vintage labels; split assignment is row-disjoint in code, so the shared -March 2017 label marks the internal cutoff rather than duplicated loans. - -The out-of-time design is adversarial to the method: the test window -spans an expansion (2018--2019) and a regime break (2020 COVID), so coverage and -the funded-set certificate are measured under a documented distribution shift -rather than on a random split that would let the model see the future. - -The design distinguishes three kinds of computation. Predictive and conformal -searches choose models, calibration, partitions, and policy families. Those -searches are frozen for this manuscript. Paper-facing reruns regenerate tables, -figures, evidence summaries, and manuscript surfaces from frozen inputs. This -separation is central to the reproducibility claim: the manuscript can evolve -without quietly reopening the policy search that selected the promoted result. -Governance keeps the two certified results distinguishable: the frozen upstream -record defines the declared return floor, while the selected-policy governance -files define the body point and its frontier. Automated claim-sync tests assert -that every body-claim number printed in the manuscript and supplement matches -those files. A validation harness additionally rebuilds the promoted conformal -intervals from the frozen PD binaries and recorded recipe and verifies the -published endpoints and coverage summaries. Gradient-boosted retraining is not -presented as the routine reproduction target across machines; retraining would -be a new research run, not the routine reproduction path for this submission. - -All primary evidence objects are represented as files with explicit ownership: model -binaries, calibration objects, conformal intervals, portfolio allocations, -tables, figures, and status reports. The anonymous submission describes the -bundle without revealing author identity. Repository and remote-storage URLs -will be disclosed according to the journal's double-anonymous and -data/code-disclosure policy. - -The body-supplement split is fixed before submission. The body keeps the CRPTO -pipeline, the alpha-to-portfolio link, the finite-grid frontier, and the -core metrics, plus the compact regret-auditability frontier. The online -supplement carries A3--A39, the conformal finalist ablation, funded-set loan -audit, tail-risk diagnostics, satisficing margins, dependence diagnostics, the CVaR/OCE -tail-constrained re-optimization (A22), the multi-distribution (A23) and -online ACI-stability (A24) diagnostics, the external economic replication -tables (A25--A34), the selected-policy frontier and funded-set audits -(A35--A39), MRM/fairness material, and reproduction commands. This keeps -the IJDS body focused while preserving the audit trail that reviewers need. - -## Multi-Dataset External Replication Protocol - -The external-replication layer is narrower than a new benchmark -campaign. We reuse the frozen CRPTO recipe--CatBoost PD, calibration, -train-only WOE/IV feature screening, Mondrian conformal intervals, and the same -bound-aware robust LP--on two credit datasets with economic fields. Prosper -contributes final-status marketplace loans with observed outcome, loan amount, -yield/rate, and a temporal OOT window [@prosperLoanData]. Freddie/Mendeley -contributes processed single-family mortgage panels derived from the Freddie Mac -loan-level ecosystem, including 12--60 month default windows and train/OOS/OOT -splits [@freddieMacSfLoanLevel; @mushava2023classimbalance]. Home Credit was -audited but not promoted because it lacks a clean investment-return and exposure -contract comparable to Lending Club, Prosper, and Freddie. - -The replication gate is fixed before inclusion: global 90% coverage must meet -target, the $\alpha=0.01$ conformal coverage must be reportable, and the LP must -return a positive robust objective when economic fields are available. This gate -does not re-promote the Lending Club champion and does not claim a new exact -funded-set theorem for every external portfolio. Its purpose is empirical: -address whether the method survives two materially different credit products. +This assumption does not follow from marginal split conformal because the +optimizer chooses the weights using $p_i$ and $u_i$. We therefore treat the +Markov expression as a secondary, assumption-conditional sensitivity. The +primary evidence is the exact replay, OOT-outcome-column-free policy ranking +conditional on the frozen conformal recipe, and observed funded-set audit. +Full proofs and sharper-assumption diagnostics are in the online supplement. # Results -The core metric table summarizes the paper-facing metrics. The calibrated PD -layer is not sold as a leaderboard model: AUC `0.7139` is sufficient only -because the downstream decision consumes calibrated probabilities, not rankings -alone. Its Brier score `0.1544` and ECE near `0.0070` are therefore as important -as discrimination. The conformal layer over-covers marginally at the reported -levels (90% coverage `0.9297`, 95% coverage `0.9664` for the conformal winner). -The portfolio layer then turns this uncertainty into an exact finite-grid -return-bound frontier. The selected policy passes the -$V \leq \sqrt{\alpha}$ certificate at the tightest reported level while keeping -zero deterministic violation. - -| Layer | Metric | Value | -|---|---:|---:| -| PD | AUC | `0.7139` | -| PD | Brier score | `0.1544` | -| PD | ECE | `0.0070` | -| Conformal | Coverage 90% | `0.9297` | -| Conformal | Coverage 95% | `0.9664` | -| Conformal | Minimum group coverage 90% | `0.9190` | -| Portfolio | Body-point robust return | `$184,832.48` | -| Portfolio | $V(\alpha = 0.01)$ | `0.035350` | -| Portfolio | $\Gamma_{\mathrm{CP}}(\alpha = 0.01)$ | `0.162616` | -| Portfolio | Markov cap at $\alpha = 0.01$ | `0.345084` | -| Portfolio | Exact alpha violation | `0.0` | -| Portfolio | Declared alpha-grid pass | `8/8` | - -: Frozen paper-facing metrics by layer. - -The exact certificate is an accounting claim. Here "exact" means the quantities -are computed directly on the frozen OOT funded set rather than approximated by a -surrogate table or visual proxy, and the deterministic part requires no -distributional assumption. The certificate's `pass` is the Markov safety check -$V(\alpha) \leq \sqrt{\alpha}$ together with zero deterministic violation -($\sum_i w_i Y_i \leq B_u(\alpha)$); it is *not* a claim of nominal -$\alpha$-coverage, which the selected funded set does not attain -($V = 0.035350 > \alpha = 0.01$). - -| $\alpha$ | $\Gamma_{\mathrm{CP}}$ | $V(\alpha)$ | $\sqrt{\alpha}$ | Markov cap | violation | pass | -|---:|---:|---:|---:|---:|---:|:---:| -| `0.01` | `0.162616` | `0.035350` | `0.10000` | `0.345084` | `0.00000` | yes | - -: Exact certificate for the selected body point. `pass` denotes -$V \leq \sqrt{\alpha}$ with zero deterministic violation, not nominal -$\alpha$-coverage. - -This makes $\Gamma_{\mathrm{CP}}$ more than a diagnostic line item. It is the -amount of conformal robustness the optimizer accepts in order to keep the funded -set inside the $\sqrt{\alpha}$-safe region. A credit reviewer can therefore read -$\Gamma_{\mathrm{CP}} = 0.162616$ as the conformal premium carried by the -selected body point, $V = 0.035350$ as the realized weighted noncoverage audit, -and the Markov cap `0.345084` as the conservative theorem-facing risk cap. - -The central empirical object is now a return-bound frontier rather than a single -winner. The consolidated frontier surface deduplicates 51,678 raw rows into -50,010 semantic policies; 27,508 policies both pass every level in the declared -alpha grid and exceed the declared return floor. The terminal endpoint search -alone evaluates 37,068 policies and 296,544 exact alpha checks, with -37,068/37,068 policies passing all eight alpha levels. - -The provenance of the frontier deserves one explicit sentence, because two -certified numbers coexist in the project's governance record. The declared -return floor `$170,464.54` is itself the realized return of the previously -certified bound-aware allocation on the same frozen chain, retained as the -frozen upstream baseline; the declared frontier is a deterministic re-evaluation -of a pre-declared finite policy grid over the *same* frozen PD model and -Mondrian conformal intervals, so no upstream object (model, calibrator, or -interval) was regenerated when the body point moved from the floor to -`$184.8K`. The earlier certificate is not discarded: it becomes the floor -that every eligible frontier policy must beat, which is why the frontier is -reported with the floor surplus rather than as a replacement champion. - -| Policy role | Source | Realized return | $V(0.01)$ | $\Gamma_{\mathrm{CP}}$ | Markov cap | Pass | -|---|---|---:|---:|---:|---:|:---:| -| Minimum Markov-cap endpoint | terminal | `$170,467.27` | `0.031875` | `0.095719` | `0.273036` | `8/8` | -| Low-cap balanced endpoint | terminal | `$171,006.20` | `0.031875` | `0.097190` | `0.274789` | `8/8` | -| Highest return under cap <= `0.30` | terminal | `$173,314.04` | `0.035875` | `0.115400` | `0.294580` | `8/8` | -| Highest return under cap <= `0.345` | micro-ext | `$184,800.41` | `0.035350` | `0.162562` | `0.344996` | `8/8` | -| Body/default balanced point | micro-ext | `$184,832.48` | `0.035350` | `0.162616` | `0.345084` | `8/8` | -| Highest return under cap <= `0.36` | micro-ext | `$186,050.73` | `0.037750` | `0.174600` | `0.358685` | `8/8` | -| Max-return economic endpoint | micro-ext | `$223,458.14` | `0.069575` | `0.457438` | `0.510753` | `8/8` | - -: Pool93 finite-grid return-bound frontier. `terminal` rows come from the -conservative endpoint sweep; `micro-ext` rows come from the body-cap local -extension around the selected point. All rows are finite-grid points, not -continuous optima. - -The table gives the manuscript its decision geometry. The body/default point is -not the highest-return point and not the tightest-bound point; it is the -balanced point selected by the declared return-bound lens. The endpoint at -Markov cap `0.273036` shows how conservative the certified frontier can become -while preserving the return floor, and the `$223.5K` endpoint shows the economic -return available when the committee accepts a looser cap. The supplement reports -the full frontier table and traceability details. - -The funded-set under-coverage remains structural rather than a calibration-draw -effect. With $n_{\mathrm{cal}} = 237{,}584$ calibration loans, the -split-conformal conditional-coverage result makes marginal coverage highly -stable around the nominal level [@vovk2005; @angelopoulos2023]. The residual -funded-set $V$ is a test-side, portfolio-selection quantity. That is why the -paper reads the safety level at $\sqrt{\alpha}$ under Assumption 1, and why the -frontier reports $V$, $\Gamma_{\mathrm{CP}}$, endpoint budget, Markov cap, and -return together instead of promoting a standalone coverage number. - -The compact reviewer checks below summarize the body-level defense. The -supplement expands the same structure into traceability and guardrail references. - -| Reviewer concern | Body answer | Primary evidence | -|---|---|---| -| "This is only a classifier." | The claim is decision auditability, not AUC leadership. | Exact funded-set certificate and A35 frontier. | -| "CP + RO already exists." | CRPTO instantiates the idea for frozen credit PD models, funded-set governance, and Lending Club payoffs. | Closest-work boundary and bound claim stack. | -| "Adaptive selection breaks coverage." | The theorem states weighted funded-set validity as an assumption and then audits the frozen selection exactly. | Assumption map, validity ladder, and A23 diagnostics. | -| "The selected policy is cherry-picked." | The selected point comes from a consolidated finite frontier with 50,010 deduplicated semantic policies and 27,508 eligible all-alpha above-floor policies. | Frontier table and governance files. | - -: Reviewer claim checks in the main manuscript. - -## Multi-Dataset External Economic Replication - -The natural generalization question after the 276,869-loan Lending Club audit is -whether the recipe still works outside the champion panel. The table below -answers that question without changing the champion: the same frozen recipe is -applied to two external credit products. Prosper is a -marketplace personal-loan panel with final statuses and a full OOT economic -candidate universe. Freddie FM48 is a collateralized mortgage panel, using the -48-month red+green default window with provided train/OOS/OOT structure. Both -pass the conformal gates and both return positive robust LP objectives. - -| Dataset | Product | Rows | Default | AUC | Cov. 90% | Cov. alpha01 | OOT cand. | Robust LP | -|---|---|---:|---:|---:|---:|---:|---:|---:| -| Prosper | Marketplace personal loans | `54,807` | `30.92%` | `0.7074` | `0.9205` | `0.9943` | `10,531` | `$199,419` | -| Freddie FM48 | Single-family mortgages | `3,173,355` | `1.45%` | `0.7839` | `0.9745` | `0.9907` | `1,396,053` | `$1,291,228` | - -: External economic replications using the frozen CRPTO recipe. The table is -generated from `crpto_tableA25_external_replication_gate.csv`; Home Credit is -archived but not promoted because it lacks a comparable economic -exposure/return contract. - -Figure @fig-external-replication summarizes the same result visually. Prosper -uses its full `10,531`-loan OOT economic universe. Freddie is evaluated on -`1,396,053` OOT economic candidates. A sparse all-candidate LP solves the full -Freddie universe and returns the same robust objective as the top screens; the -worst funded loan has rank `551`, with zero funded loans outside the top-250,000 -screen. The supplement reports this as an exhaustiveness audit rather than -as an unverified shortlist caveat. - -![External CRPTO replications preserve the predeclared global conformal gates and produce positive robust LP value on two materially different credit products.](../reports/crpto/figures/crpto_fig22_external_replication.png){#fig-external-replication width="94%" fig-alt="Two-panel external replication figure. The left panel shows 90 percent and alpha 0.01 coverage for Prosper and Freddie FM48 above target lines; the right panel shows positive robust LP objective values and OOT candidate counts."} - -The external layer also surfaces a result a single-dataset champion cannot show. -The signed price of robustness--using the same convention as the Lending Club -field, $(\text{nonrobust}-\text{robust})/\text{nonrobust}$--is a *positive* -premium under frozen application, and across the four frozen external -applications it is ordered by panel default rate (Table @tbl-price-of-robustness, -Figure @fig-price-scaling). Within Freddie, the high-default red segment pays -more than green; across datasets, Prosper's `30.92%` default panel pays the -largest premium. This is a pattern across two datasets -- three Freddie -default-window segments plus Prosper -- not a scaling law: it is consistent with -the mechanism (higher default risk widens the conformal intervals, so the robust -worst case discounts more return, and discrimination (AUC) does not order the -premium on its own), but four points cannot establish a general law. The -measured summary is intentionally modest: in these external applications, the -conformal robust layer is economically bounded under blind application, with a -single-digit to low-double-digit premium. The Lending Club claim is handled -separately through the exact return-bound frontier rather than through this -external price-scaling table. -That closes the external-replication claim at the right level: the recipe -transfers as an economic audit protocol, while the exact funded-set certificate -remains the Lending Club object. - -| Application | Panel default | Price of robustness | -|---|---:|---:| -| Freddie FM48 (green) | `0.58%` | `+1.00%` | -| Freddie FM48 (combined) | `1.45%` | `+1.09%` | -| Freddie FM48 (red) | `2.97%` | `+2.37%` | -| Prosper final-status | `30.92%` | `+9.46%` | - -: Price of robustness by external application, ordered by panel default rate, -from `crpto_tableA34_price_of_robustness_cross_dataset.csv`. {#tbl-price-of-robustness} - -![Across the four frozen external applications, the price of robustness is ordered by panel default risk.](../reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png){#fig-price-scaling width="78%" fig-alt="Line chart on a log-scale default-rate axis: the price of robustness rises from +1.00 percent to +9.46 percent across external applications."} - -# Robustness and Comparators - -The first robustness concern is temporal leakage. CRPTO addresses it through -out-of-time splits, temporal backtesting, and a strict distinction between -calibration, test, and paper-facing outputs. The paper does not claim that a -Lending Club static panel substitutes for future originations after the retail -platform closed. It claims that within the available historical panel, the -promoted policy survives the documented temporal validation design. The strict -temporal holdout in supplement A9 reports both temporal slices passing the exact -check, strengthening that validation claim without reopening the champion. - -The second concern is whether conformal uncertainty is doing decision work or -only adding conservative decoration. The answer is visible in the portfolio -frontier. Policies are evaluated by return, exact alpha pass/fail, weighted -miscoverage, and $\Gamma_{\mathrm{CP}}$; the promoted point is selected because it earns -the highest robust return inside the exact feasible region. This differs from -a workflow where conformal intervals are plotted after the optimizer has -already chosen a point-PD allocation. - -The supplement also carries the reviewer-facing robustness checks: nested -holdout, strict temporal holdout, exact evaluation of conformal finalists, -uncertainty-set baselines, and the selected-policy finite-grid frontier. -Diagnostics that depend on the selected funded-loan composition are split -explicitly: selected-allocation checks stay diagnostic, while legacy -tail-frontier tables remain diagnostic machinery rather than body selectors. - -The third concern is whether a decision-focused or SPO+ model would be a -stronger baseline. CRPTO treats SPO+ as an important comparator, but not as the -same governance object. Decision-focused training can reduce regret relative -to an optimization loss, while CRPTO prioritizes calibrated uncertainty, -traceable risk controls, and exact funded-set checks. The manuscript therefore -does not claim to dominate SPO+ on every regret metric; it claims a different -auditability/economic trade-off. - -## Regret-Auditability Frontier - -The SPO+ comparator makes the trade-off sharp. In the committed A19/Fig. 15 -results, SPO+ reduces mean regret from `0.425896` to `0.216837`, a `49.09%` -improvement over the two-stage baseline (Wilcoxon `p = 1.39e-164`). A later -PyEPO 1.3.7 paired rerun independently reports the same conclusion -under a slightly different protocol (`0.358073` to `0.184366`, `48.51%`; -Wilcoxon `p = 3.80e-163`), so we treat it as a curated closeout note rather -than as the numeric source for A19. The CRPTO robust point has higher decision -regret (`0.947429`) because it is not trained to minimize regret; it is -constructed to expose and control predictive uncertainty before funding. The -frontier is therefore not a single leaderboard. It asks what the method buys -besides regret: temporal coverage above target, an exact funded-set -$\alpha = 0.01$ pass, and a finite-grid return-bound frontier. This is the cleanest -comparator story in the paper. SPO+ answers "how much regret can training -remove?" CRPTO answers "what can a reviewer verify after a calibrated PD model -is frozen?" Those are complementary questions, and the frontier makes the -trade-off explicit rather than burying it in a single score. - -| Method | Mean regret | Regret delta vs. two-stage | Realized funding value | Verifiable risk controls | +## Exact 90% Conformal Evidence + +At the active target $\alpha=0.10$, exact OOT coverage is `0.934836`, average +interval width is `0.788879`, minimum score-partition coverage is `0.926310`, +minimum letter-grade coverage is `0.926797`, and `51.7873%` of upper endpoints +equal one. The intervals are conservative and broad, which is expected for a +binary outcome on the probability scale. + +The exact alpha sensitivity explains why the paper does not headline a 99% +interval. At target alpha `0.01`, empirical coverage is `0.996720`, but average +width is `0.988215` and `93.5424%` of upper endpoints equal one. Such endpoints +carry almost no ranking information for a portfolio. The selected 90% level is +the frozen recipe's reference level and preserves materially more decision +resolution. A35 reports the complete sensitivity from alpha `0.01` to `0.20`. + +## Pre-OOT Selector and Decision Audit + +November selects the midpoint policy from five eligible rows. Applying the +same outcome-free rule to December again selects `linear-005`; policy identity +is therefore stable across the two calibration months. Outcomes are opened +only after that replay. + +| December policy | Funded | Realized return | Weighted default | Miscoverage | $B_u$ | +|---|---:|---:|---:|---:|---:| +| Selected 50/50 CRPTO | `193` | `$53,313.05` | `0.145650` | `0.124925` | `0.262082` | +| More-conservative 75% blend | `191` | `$38,379.50` | `0.155250` | `0.134525` | `0.203504` | +| Point-PD matched-$\tau$ | `169` | `$89,732.35` | `0.185650` | `0.058300` | `0.888071` | + +: Independent December 2017 post-selection decision audit. + +The selected policy remains below its operational tolerance +(`0.145650 < 0.17`) but misses nominal funded-set coverage +(`V=0.124925 > 0.10`). Its deterministic accounting right-hand side is +`B_u+V=0.387007`. Thus an outcome-free, month-stable selector does not create +selected-set conformal validity. This negative audit is part of the result: it +motivates the two-layer reporting of deterministic endpoint exposure and +observed miscoverage instead of relabeling marginal coverage as a portfolio +guarantee. + +## Full OOT Funded-Set Audit + +The fixed midpoint policy allocates the full `$1M` budget across 308 loans. Its +expected point-PD objective is `$168,271.56`; realized return is `$179,327.59`. +The weighted default rate is `0.039375`, and weighted miscoverage is `0.036875`. + +| Quantity | OOT value | +|---|---:| +| Weighted point PD | `0.081949` | +| Weighted midpoint score | `0.170000` | +| $\Gamma_{\mathrm{CP}}$ | `0.176102` | +| $\Gamma_{\mathrm{int}}$ | `0.088051` | +| $\Gamma_{\mathrm{res}}$ | `0.088051` | +| Endpoint budget $B_u$ | `0.258051` | +| Observed accounting bound $B_u+V$ | `0.294926` | +| Conditional Markov threshold | `0.574279` | + +: Exact full-OOT audit of the selected policy. + +The observed weighted outcome `0.039375` is well below the exact accounting +right-hand side `0.294926`. Under the additional weighted-validity assumption, +the `0.574279` event threshold has probability bound +$\sqrt{0.10}=0.316228$. This loose probability statement is not interpreted as +a direct default cap. The operational controls are $\tau=0.17$, the midpoint +score, and the exact funded-set diagnostics. + +The primary fixed-allocation bootstrap resamples 31 origination-month clusters +and gives a 95% return interval of `$163,421.14`--`$193,551.65` from 5,000 +draws. A loan-level sensitivity gives +`$162,706.17`--`$193,924.74`. Neither scheme resamples the model, conformal +recipe, policy selector, or optimizer; both are contribution-level stability +diagnostics rather than full-pipeline confidence intervals. The month-level +scheme is primary because the temporal results reject a naive independence +story across loans. + +## Matched Point-PD and More-Conservative Comparators + +| Policy | Funded | Realized return | Weighted default | Miscoverage | $B_u$ | Conditional threshold | +|---|---:|---:|---:|---:|---:|---:| +| Selected 50/50 CRPTO | `308` | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | +| More-conservative 75% blend | `312` | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | +| Point-PD matched-$\tau$ | `225` | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | + +: Full-OOT matched decision comparison. + +Relative to point PD, selected CRPTO gives up `$17,041.55`, or `8.678%` of +realized return. Weighted default falls by `7.9025` percentage points, +miscoverage falls by `0.5025` percentage points, and the endpoint-plus-Markov +threshold falls by `66.3266` percentage points. The default contrast is much +larger than the miscoverage contrast: conformal robustness changes which loans +are funded, but it does not create a dramatic selected-set coverage gain. + +The 75% blend reveals the remaining internal trade-off. It lowers default by +`0.35` percentage points and the threshold by `0.057655` relative to the +selected midpoint, but costs another `$6,388.08` in realized return. The +calibration selector chooses the midpoint because it is the highest expected- +objective candidate under the declared screen, not because it dominates every +risk metric. + +## Temporal Heterogeneity + +The full-panel average hides substantial regime dependence. + +| Period | CRPTO return | Point-PD return | CRPTO default | Point-PD default | |---|---:|---:|---:|---:| -| Two-stage baseline | `0.425896` | `0.0%` | not certified | `0/3` | -| SPO+ | `0.216837` | `49.09% lower` | not certified | `0/3` | -| CRPTO robust | `0.947429` | `122.46% higher` | `$184.8K` | `3/3` | - -: Regret-auditability frontier. Regret is the A19/PyEPO synthetic -decision-loss scale, not realized dollar return on the `$1M` funded portfolio. - -The regret column must be read with its protocol in mind, and the last two -columns prevent a one-dimensional reading. Mean regret comes from a separate -decision-regret experiment (A19/PyEPO) run on small synthetic optimization -instances (50 items, budget 15, five seeds), which scores each method on a -normalized decision-loss scale rather than on the `$1M` funded portfolio. That -experiment is not the same object as the funded-set economics: it -isolates training-time decision quality, so SPO+ is the low-regret method by -construction because it is trained to minimize exactly that loss. The -favorable price of robustness reported above and the higher CRPTO regret here -are therefore not in tension; they are two different measurements (a real -`$1M` funded set versus a synthetic regret benchmark). The right-hand columns -report what the credit decision actually delivers: only CRPTO produces a -budgeted funded set with a certified realized return (`$184.8K` on the -`$1M` budget) and the three verifiable risk controls (exact $\alpha$-safe pass, -weighted-miscoverage audit, and finite-grid return-bound frontier). The two regret-trained -comparators optimize a loss surface but do not emit an auditable funded-set -certificate. The regret comparison is therefore about the synthetic benchmark -task, not the quality of the funded loans in the `$1M` credit portfolio. - -![The regret-auditability frontier shows the paper's trade-off in one panel: SPO+ is the low-regret corner, while CRPTO robust is the auditable-risk-control corner with all three verifiable checks passing.](../reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png){#fig-regret-auditability width="72%" fig-alt="Scatter plot comparing two-stage, SPO+, and CRPTO robust by mean decision regret and number of verifiable risk-control checks passed."} - -CRPTO is also close in spirit to a recent line that gives distribution-free, -finite-sample guarantees jointly on miscoverage and decision regret, and that -traces a miscoverage--regret Pareto frontier for robust predict-then-optimize -policies [@zhou2025credo; @zhou2026creme]. That work is the general theory of -the frontier in @fig-regret-auditability: it shows how to calibrate a robustness -level against a cost--risk preference for an abstract optimization family. CRPTO -is the complementary, applied object. It does not propose a new frontier -estimator; it instantiates one corner of that frontier on a frozen, -production-style credit-risk model, with a named conformal robustness premium -$\Gamma_{\mathrm{CP}}$, an exact funded-set certificate, grade-level audits, and -model-risk lineage that a credit committee can inspect. A reviewer who knows the -general theory should therefore read CRPTO not as a competing estimator but as -the credit-decision instantiation that connects that theory to an auditable -Lending Club funded set. - -## Tail Risk and Distribution Robustness - -Two reviewer questions deserve a body-level answer rather than an appendix-only -one: what does the decision give up on the tail, and does its coverage hold once -the evaluation is sliced by grade? The return-bound frontier answers the first -question directly: lower Markov caps are available at lower return, while the -selected policy sits on the declared frontier. The supplement then checks the -funded-grade mix, selected-allocation tail repricing, dependence sensitivity, -and fixed-allocation bootstrap interval. These rows explain the selected -policy's risk profile; they do not add a hidden CVaR/OCE or bootstrap selector. - -| Reviewer question | Body answer | Boundary | -|---|---|---| -| Is the point only high-return? | The frontier shows safer lower-return choices and higher-return looser-cap choices. | Finite grid, not a continuous optimum. | -| What loans does it fund? | The supplement reports funded exposure by grade. | Business mix, not protected-class fairness certification. | -| What happens in the tail? | The selected allocation is repriced under LGD and tail summaries. | Risk profile only; tail risk is not the selector. | -| Does dependence change the bound? | Cluster sensitivity recomputes tighter assumptions. | Sensitivity only; Markov remains the body theorem. | -| Is the return estimate fragile? | A fixed-allocation bootstrap reports contribution intervals. | Does not resample the model, solver, or search. | -| Does coverage survive slices? | Grade, distribution, and online-style checks stress the frozen intervals. | Diagnostics, not universal conditional or live coverage. | - -: Tail-risk and distribution-robustness checks for the selected decision. - -The second question is distribution robustness across grades. On the frozen -intervals, the worst per-grade 90% coverage is grade E at `0.9004`, still above -the `0.90` target. Supplement A23 reports marginal coverage `0.9293` on its -multi-distribution evaluation slice, while the promoted interval summary in -Table 3 reports `0.9297`; these are distinct cuts through the same -intervals. No grade falls below target, so the conservative marginal -coverage is not hiding a -failing segment; the thinnest grade$\times$vintage cells are where a future -group-weighted or multi-distribution recalibration would matter, and we mark that -as future work rather than a present guarantee. - -## Managerial Implication - -For a credit-risk committee, CRPTO turns a model into a decision -conversation. The committee can pick a risk cap $\tau$, inspect how a policy -parameter $\gamma$ changes the funded set, read $\Gamma_{\mathrm{CP}}$ as the conformal -robustness premium paid by that funded set, and compare $V(\alpha)$ with the -stated bound tolerance. The method therefore supports a practical question: -whether the realized economic return justifies the signed price of robustness, -which on this evaluation is favorable rather than a cost. -If the committee wants lower regret, the SPO+ corner is visible; if it wants -stronger validity language, the validity ladder states the new calibration -protocol that would be required. That separation is the managerial value of the -paper. - -The online supplement contains the robustness package: nested holdout, -segment-period sensitivity, decision-aware conformal selector checks, -synthetic-shift diagnostics, conformal finalist exact evaluation, tail-risk -OCE/CVaR diagnostics, satisficing margins, dependency clusters, bootstrap -funded-set metrics, budget/LGD/cap sensitivity, and finite-grid frontier -summaries by policy family, plus the A19 regret-auditability frontier, A20 -tail-risk diagnostic audit, A21 cluster-bound tightening audit, A22 CVaR/OCE -tail-constrained re-optimization, and A23--A24 multi-distribution and online -(ACI) conformal-stability diagnostics, plus A25--A34 external economic -replication, exhaustiveness, interval, subperiod, and sensitivity audits on -Prosper and Freddie/Mendeley, and A35--A39 selected-policy frontier, -composition, tail-risk, concentration, and bootstrap audits. - -# Reproducibility and Companion - -The project is built as an executable research bundle. Source code, manuscript -files, DVC metadata, tables, figures, and status reports are versioned together; -heavy data and model files live outside Git and are verified by hashes. The -paper-facing commands regenerate tables, figures, evidence summaries, and -rendered manuscript surfaces from frozen inputs; protected model-search stages -are not rerun as part of ordinary paper revision. - -Governance keeps the two certified results distinguishable. The frozen upstream -record defines the declared return floor, while the selected-policy governance -files define the body point and its frontier. Automated claim-sync tests assert -that every body-claim number printed in the manuscript and supplement matches -those files, so the two records cannot silently diverge. - -The certificate chain carries a stronger but narrower property: a validation -harness rebuilds the promoted conformal intervals from the frozen PD binaries -and recorded recipe and verifies the published endpoints and coverage summaries. -Gradient-boosted retraining is not presented as the routine reproduction target -across machines; retraining would be a new research run, not the routine -reproduction path for this submission. - -For double-anonymous review, this manuscript omits author-identifying URLs and -uses neutral language around repository ownership. The data/code companion can -be disclosed in the cover letter, supplement, or post-review reproducibility bundle -according to the venue policy. At acceptance, the reproducibility package is -designed to include public code, Quarto sources, pointers for processed data -and model files, raw Lending Club source instructions rather than secrets or -redistributed credentials, and the commands used to regenerate paper tables, -figures, and rendered manuscript surfaces. The important reproducibility -property is that the manuscript does not depend on hidden spreadsheet edits: -the reported numbers come from versioned evidence and guardrail tests. - -# Discussion - -CRPTO is useful precisely because it stays close to the operational reality of -credit-risk analytics. Many institutions already have calibrated PD models and -portfolio policies. Replacing them with an end-to-end decision-focused learner -may be scientifically attractive but organizationally difficult. CRPTO offers a -middle path: keep the predictive model auditable, quantify uncertainty with a -finite-sample conformal layer, and make the optimizer pay attention to the -upper end of plausible default risk. - -The external replications also change how to read the price of robustness. -Across the frozen external applications, the premium is ordered by panel default -risk rather than by discrimination. That pattern reframes robustness as a -panel-specific premium to be measured, not a fixed toll to be assumed. It also -tempers the contribution: the favorable Lending Club value reflects champion -selection, so the transferable claim is bounded measurement under blind -application, not a universal free lunch. - -The limits are equally important. CRPTO does not prove that any one public -dataset is a universal proxy for modern credit origination, even after the -Prosper and Freddie/Mendeley replications. Those external runs strengthen the -empirical defense across credit products, but they remain static historical -replications, not new exact funded-set certificates and not a prospective -deployment. The paper also does not claim legal -fair-lending certification because the public data lack direct protected -attributes. It does not assert that robust conformal policies dominate all -decision-focused learners on regret, and it does not treat synthetic shift or -external static panels as substitutes for live post-deployment monitoring. Its -conformal guarantee is marginal or partitioned by the chosen Mondrian design, -not exact conditional coverage for every borrower profile. The external panels -make this concrete: on Freddie the all-group minimum coverage is driven by tiny -sparse Mondrian cells, and the high-default red segment misses the strict -$\alpha = 0.01$ gate at `0.9850`; both are reported as sensitivity -evidence rather than promoted as conditional guarantees. Finally, this is not -an online deployment study: there are no new post-2020 Lending Club retail -originations, no live monitoring loop, and no end-to-end utility-directed -conformal learner replacing the frozen PD model. - -The next research directions are clear and are now separated into paper -improvements versus new CRPTO protocols. The current paper includes the safe -journal-strengthening package: OCE/CVaR as a tail-risk audit, satisficing as -margin evidence, SPO+ as the low-regret corner of the regret-auditability -frontier, and dependence as a formal caveat. The larger scientific upgrades are -useful in the present manuscript only when they sharpen this boundary. Table -@tbl-upgrade-map states that boundary directly. - -| Upgrade path | What the current paper can claim | What would require a new result | -|---|---|---| -| Tail-aware selection | A20--A22 and A37 show the tail trade-off and selected-allocation tail profile. | A predeclared CVaR/OCE selector with its own exact funded-set audit. | -| Prospective selection | Nested holdout and the finite declared grid reduce post-selection ambiguity. | A fully prospective search/evaluation split after all policy rules are frozen. | -| Multi-distribution or online validity | A23--A24 diagnose grade, distribution, and vintage stress on frozen intervals. | A new calibration protocol that targets group-weighted, multi-source, or live sequential validity. | -| Decision-focused conformal learning | A19 shows SPO+ as the low-regret comparator and CRPTO as the auditable corner. | An end-to-end learner that jointly optimizes decision loss and emits a conformal funded-set certificate. | - -: Scientific upgrade map for the current CRPTO paper. {#tbl-upgrade-map} - -Optimized OCE/CVaR objectives, online conformal methods, multi-distribution -conformal validity, utility-directed or decision-theoretic conformal variants -[@cortesgomez2025utility; @lekeufack2023cdt], causal variants, broader -asset-class panels, and prospective multi-period origination studies are all -valuable. They become stronger paper claims only with new protocols, data, or -proofs. In the current submission, their role is to make the frozen CRPTO -contribution easier to locate: an auditable post-hoc predict-then-optimize -certificate, not a universal decision-learning framework. +| 2018H1 | `$92,530.73` | `$118,101.99` | `0.106703` | `0.190825` | +| 2018H2 | `$156,185.51` | `$95,603.58` | `0.026725` | `0.236728` | +| 2019H1 | `$123,590.69` | `$144,281.46` | `0.077325` | `0.170275` | +| 2019H2 | `$110,251.95` | `$256,966.20` | `0.103250` | `0.023775` | +| 2020+ | `$99,689.54` | `$218,629.14` | `0.083775` | `0.016900` | + +: Fixed-policy temporal stress evaluation; each row uses a fresh `$1M` budget. + +CRPTO is economically and statistically attractive in 2018H2, when the +point-PD portfolio concentrates heavily in realized defaults. In 2019H2 and +2020+, the point-PD policy earns much more and defaults less. The honest result +is therefore a full-period return-risk trade-off with temporal heterogeneity, +not a universal robustness premium. This pattern also cautions against treating +a static conformal guardrail as a substitute for live recalibration. + +## Funded-Set Composition + +The selected allocation is concentrated in grades C and D: they represent +`31.36%` and `58.68%` of funded exposure. Grade F is only `1.30%` of exposure +but has a high realized default rate (`0.269231`), illustrating why aggregate +metrics should be accompanied by composition. A38 reports every funded grade +and reconciles to the full `$1M` allocation. Because Lending Club does not +provide the protected attributes needed for a legal fair-lending audit, this is +a business-risk composition check, not a fairness certification. + +# Robustness, Comparators, and Implications + +The online supplement contains the evidence that helps a reviewer challenge +the one active method without turning the paper into several methods: exact +alpha saturation (A35), all nine selector cells (A36), temporal evaluation +(A37), grade composition (A38), fixed-allocation bootstrap (A39), and matched +comparators (A40). Earlier OCE/CVaR, dependence, online-style, SPO+, Prosper, +and Freddie/Mendeley analyses are retained as diagnostics or external context. +They do not select the active policy or create additional claims. + +For a credit committee, the useful control is not the Markov threshold by +itself. The committee first chooses a conventional conformal level, a +transparent blend $\gamma$, and a portfolio tolerance $\tau$. It can then read +the economic cost, endpoint exposure, realized default, and temporal behavior +together. In this application the midpoint rule has a clear interpretation: +half of the conformal premium enters the enforceable risk score and half remains +visible as residual endpoint exposure. + +The matched baseline turns that interpretation into a decision. A committee +that accepts the full-period evidence pays `8.678%` of realized return for a +`7.9025` percentage-point reduction in weighted default. A more conservative +committee can move to the 75% blend and pay an additional `$6.4K`. A committee +that emphasizes the 2019H2 or 2020 regimes should reject the static CRPTO rule +or require a new temporal recalibration protocol. The method exposes that +choice rather than hiding it behind a single score. + +# Reproducibility and Limitations + +The research bundle versions code, manuscript sources, configurations, tables, +figures, and governance files together. Heavy data and model artifacts are +stored separately and checked by hashes. The active evidence builder regenerates +A35--A40 from the frozen exact-alpha and policy outputs. Claim-sync tests verify +that the body, supplement, and official IJDS LaTeX manuscript share the same +policy, selector counts, and numeric anchors. + +Several limitations bound the contribution. First, Lending Club retail +origination ended in 2020; the panel is historical and cannot establish live +post-2020 performance. Second, the final tagged selector is outcome-free with +respect to OOT ranking, but earlier project development inspected the same +static OOT corpus. The evaluation is therefore retrospective, not a new +prospective holdout. Third, the conformal intervals are broad because the +outcome is binary; at 90%, more than half of OOT upper endpoints equal one. +Fourth, marginal or Mondrian coverage does not imply validity under +optimizer-selected funded weights. The independent December audit makes this +visible: the same policy is reselected without outcomes, yet funded-set +miscoverage is `0.124925`. The Markov result is therefore conditional on an +explicit weighted-validity assumption and is absent from policy selection. +Fifth, temporal slices show that the point-PD comparator can dominate both +return and default. Finally, public data do not support a legal fair-lending +certification or a causal interpretation of policy contrasts. + +These limitations suggest a focused next step rather than a larger current +paper: a genuinely prospective or formally selection-valid protocol that +freezes conformal and policy choices before a new evaluation period +[@farinhas2024nonexchangeable_crc; @hegazy2025valid_selection_conformal_sets; +@zhou2026creme]. +It is not required to interpret the present retrospective decision audit. + +For double-anonymous review, author-identifying repository URLs are omitted +from the manuscript. The code/data companion and artifact-access instructions +can be disclosed under the journal's policy without embedding credentials or +identity in the anonymous PDF. # Conclusion -Can finite predictive uncertainty change a funding decision in a way a reviewer -can audit? CRPTO's answer is yes, provided the statistical boundary is stated -and the decision record is frozen. On the Lending Club out-of-time panel the -selected policy earns `$184,832.48` on a `$1M` budget while passing -the exact empirical $\alpha = 0.01$ funded-set audit, and it lies on a declared -finite-grid return-bound frontier with 50,010 deduplicated semantic policies and -27,508 all-alpha above-floor policies rather than at a single lucky point. The external -Prosper and Freddie/Mendeley replications show that the recipe travels beyond -the Lending Club panel and expose an economically interpretable regularity: the -price of robustness is a bounded premium ordered by panel default risk across -the frozen external applications. The contribution is scoped as an auditable post-hoc decision certificate, -not a new end-to-end learner or a live-deployment study, and every reported -number is regenerable from frozen evidence. - -# References +CRPTO shows how a frozen credit model can become an auditable portfolio +decision without adding a maze of policy variants. An exact 90% conformal +replay produces $u_i$; the midpoint $q_i=(p_i+u_i)/2$ constrains risk; and a +nine-cell November selector fixes $\tau=0.17$ under the deterministic +$B_u\le0.28$ screen. December independently selects the same rule, then misses +nominal funded-set coverage; this is why CRPTO reports observed miscoverage +rather than claiming that policy stability implies conformal validity. +On the full OOT panel, the policy earns `$179,327.59`, with weighted default +`0.039375`, miscoverage `0.036875`, $\Gamma_{\mathrm{CP}}=0.176102$, +$\Gamma_{\mathrm{res}}=0.088051$, endpoint `0.258051`, observed accounting +bound `0.294926`, and conditional Markov threshold `0.574279`. Against the +matched point-PD allocation, it pays `8.678%` of realized return for a +`7.9025` percentage-point default reduction. Temporal reversals keep the claim +narrow: CRPTO is an inspectable retrospective return-risk guardrail, not a +universal winner, prospective deployment guarantee, or new credit-scoring +leaderboard. diff --git a/paper/README.md b/paper/README.md index ee051e6..a494001 100644 --- a/paper/README.md +++ b/paper/README.md @@ -34,20 +34,20 @@ the submission-shaped versions are written. IJDS disclosure form. - `submission/SCHOLARONE_FINAL_CHECKLIST.md`: final upload/proof checklist. -Selected P2/P3-inspired diagnostics are part of the current paper/journal pack: -regret-auditability, OCE/CVaR tail risk, robust satisficing margins, -multi-distribution diagnostics, online replay diagnostics, the pool93 A35 -finite-grid frontier, the A36 funded-set grade audit, the A37--A39 selected -pool93 tail-risk, cluster-bound, and bootstrap audits, and external Prosper/Freddie economic -replications. Tail-risk row-level repricing is supplement evidence for the -selected pool93 allocation, not a hidden promotion criterion. Prospective -live online validation, causal variants, new multi-dataset protocols beyond the -frozen Prosper/Freddie replications, production, and package tracks remain -future work. +The active paper has one method: exact 90% conformal replay, the midpoint +guardrail `q=(p+u)/2`, `tau=0.17`, and a nine-cell November selector under +`B_u<=0.28`. A35 is the exact-alpha audit, A36 is the split selector/December +audit, A37 is temporal evaluation, A38 is letter-grade composition, A39 is the +month-cluster bootstrap with loan-level sensitivity, and A40 is the matched +point-PD comparison. OCE/CVaR, SPO+, satisficing, online-style checks, +and Prosper/Freddie replications remain supplement diagnostics; they do not +select or redefine the midpoint policy. Prospective validation, causal variants, +live recalibration, production, and package tracks remain outside the claim. ## Render Commands -```bash +```powershell +just ijds-evidence just paper-ijds just paper-ijds-supplement just paper-submission diff --git a/paper/ijds.css b/paper/ijds.css index f2c21a4..134aee5 100644 --- a/paper/ijds.css +++ b/paper/ijds.css @@ -18,3 +18,23 @@ overflow-x: auto; } } + +@media print { + html, + body, + main.content, + #quarto-appendix { + height: auto !important; + min-height: 0 !important; + margin-bottom: 0 !important; + padding-bottom: 0 !important; + } + + #quarto-content { + grid-template-rows: [content-top] auto [content-bottom] 0 [page-bottom] !important; + } + + #quarto-appendix { + break-after: avoid-page; + } +} diff --git a/paper/submission/CLAIM_AUDIT_MATRIX.md b/paper/submission/CLAIM_AUDIT_MATRIX.md index dbdbe7b..fc792fd 100644 --- a/paper/submission/CLAIM_AUDIT_MATRIX.md +++ b/paper/submission/CLAIM_AUDIT_MATRIX.md @@ -1,35 +1,49 @@ -# CRPTO Claim Audit Matrix +# IJDS Claim Audit Matrix -This matrix is the pre-submission reviewer-defense layer. It maps each visible -claim to evidence, artifact provenance, and the boundary that prevents -overclaiming. +This matrix is the editorial guardrail for the active calibration-selected +midpoint policy. Numeric authority is +`ijds_policy_governance.json` plus A35--A40. -| Claim | Body location | Evidence/artifact | Reviewer risk | Boundary language | -|---|---|---|---|---| -| CRPTO is a data-science decision method, not a classifier leaderboard. | Abstract, Introduction, Related Work. | Pipeline Figure 1, exact certificate table, funded-set audit. | Desk screen reads it as applied ML. | State the central object as an auditable conformal-robust decision certificate. | -| The predictive input is frozen and calibrated. | Method, Results, Supplement Appendix E. | `models/pd_canonical.cbm`, `models/pd_canonical_calibrator.pkl`, Table 0 metrics, E3/E4 PD stability diagnostics. | Reviewer asks whether results depend on hidden retraining. | PD artifact is consumed, not re-searched, by paper renders; E3/E4 are non-promoted T1 diagnostics, not a new champion. | -| The conformal layer is conservative on OOT data. | Method, Results, Supplement A23-A24. | `conformal_intervals_mondrian.parquet`, coverage metrics, group audits. | Reviewer expects conditional coverage. | Claim marginal/Mondrian coverage; stronger local/multi-distribution claims are future work. | -| The 90% intervals are useful despite large raw width. | Method, funded-set audit, Tables A35-A39. | Average width `0.7842`, Winkler `1.1107`, and pool93 body-point `Gamma_CP=0.162616` after allocation; A37--A39 profile tail risk, concentration, and fixed-allocation bootstrap uncertainty from the selected allocation. | Reviewer says intervals are too wide on the `[0,1]` PD scale. | Width is not promoted as a standalone utility metric; the decision layer uses upper-endpoint ordering, funded-set miscoverage, and the conformal premium inside an exact portfolio frontier. | -| The promoted Lending Club body point passes the alpha-grid funded-set audit. | Results, Supplement A35-A39. | Body point: return `$184,832.48`, `V=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084`, zero violation, `8/8` alpha pass; A36 shows funded exposure by grade; A37 reports baseline LGD tail repricing; A38 shows cluster bounds remain looser than Markov; A39 reports fixed-allocation bootstrap return interval `$167,963.20`--`$198,650.47`. | "Exact" may be misunderstood as universal validity. | Exact means direct accounting on frozen funded-set outputs (Theorem 1(i)); statistical interpretation uses weighted funded-set validity, stated as Assumption 1 with full proof in Supplement Appendix A. A39 is an empirical fixed-allocation interval, not a conformal guarantee. | -| The result is not a single lucky point. | Results, Supplement A35. | Consolidated pool93 frontier: `50,010` deduplicated semantic policies, `27,508` eligible all-alpha above-floor policies; terminal endpoint search: `37,068/37,068` all-alpha passers. | Reviewer asks whether one selected policy was cherry-picked. | The frontier is a declared finite policy-grid surface, not all possible continuous policy values and not a future-selection guarantee. | -| Prosper/Freddie show transfer of the recipe. | Results, Supplement A25-A34. | External replication gate, candidate sensitivity, all-candidate LP exhaustiveness. | Reviewer reads them as new champion certificates. | They are frozen external economic replications and exhaustiveness audits only. | -| The price of robustness is economically interpretable. | Results, Discussion. | Lending Club price `-10.56%`; external premiums `+1.00%` to `+9.46%`, ordered by panel default rate across the four frozen external applications. | Sign convention, selected-vs-blind confusion, or over-reading as a statistical scaling law. | Lending Club is selected; external panels are blind frozen applications; describe the pattern as ordered in these applications, not as a general law. | -| SPO+ is a comparator, not a replacement champion. | Robustness and Comparators. | A19/Fig. 15 committed regret-auditability artifact. | Reviewer says CRPTO has higher regret. | SPO+ optimizes synthetic regret; CRPTO emits funded-set risk controls. | -| Tail-risk alternatives exist and the selected point has been repriced. | Tail Risk section, Supplement A20-A22 and A37-A39. | CVaR/OCE and tail-constrained challenger tables; pool93 body repricing has baseline LGD return `$184,832.48`, realized CVaR95 `0.276211`, decision-time CVaR95 `0.218140`, no cluster-bound threshold tighter than Markov, and fixed-allocation bootstrap diagnostics. | Reviewer asks why not select lowest-tail policy. | The body selector is the finite-grid return-bound point; tail-risk/bootstrap tables are documented trade-offs and selected-point diagnostics, not hidden promotion criteria. | -| Fairness/MRM claims are limited. | Supplement Appendix D, Discussion. | Proxy/intersectional diagnostics and MRM scope. | Reviewer asks for statutory fair-lending proof. | Public data lack direct protected attributes; no legal certification claim. | -| The paper is reproducible. | Body design paragraph, Supplement E, submission docs. | `just smoke`, `just validate-champion`, DVC metadata, lockfile, manifest. | Reviewer asks how to reproduce without secrets. | Raw-data instructions and DVC pointers are disclosed through the accepted-paper package or an editor-approved review bundle. | -| The certificate chain is exactly recomputable from frozen artifacts. | Body design paragraph in the official `.tex`, Supplement E, submission docs. | Drift harness `tests/test_models/test_conformal_mapie_drift.py` (zero max abs diff per loan and per Mondrian cell, identical re-learned floor multipliers); `scripts/rebuild_test_predictions_from_frozen.py` hard-asserts score/interval identity. | Reviewer reads "reproducible" as including GBM retraining. | The verified property is prediction-to-decision: frozen PD binaries through conformal intervals to the certificate, under the locked dependency stack. Gradient-boosted retraining is not bit-reproducible, which is why the predictive layer ships as a hash-verified frozen binary; E3/E4 support this choice without becoming routine reruns. | +| Claim | Evidence | Main reviewer risk | Defensible wording | +|---|---|---|---| +| The 90% conformal recipe is replayed exactly. | A35; stored endpoint replay max error `6.67e-16`. | "Exact" is mistaken for universal conditional validity. | Exact refers to numerical reconstruction of the frozen finite-sample recipe. Coverage remains marginal/Mondrian under its assumptions. | +| The 99% setting is not decision-useful here. | A35: width `0.988215`; `93.5424%` of upper endpoints equal one. | Reviewer thinks 90% was chosen only to improve economics. | The 90% level is the frozen recipe's reference level and preserves materially more ranking resolution; alpha sensitivity is fully reported. | +| The final policy is simple. | `q=(p+u)/2`, `tau=0.17`; one linear policy family. | Complexity or hidden nonlinear logic. | Point PD prices expected loss; the midpoint is used only in the risk constraint. | +| Final ranking does not use OOT outcomes or statistical assumptions. | A36; outcomes stored separately from a 12-column frame, nine candidates, five eligible, zero outcome/Markov fields. | Historical OOT inspection makes "untouched holdout" false. | Call it a retrospective lockbox replay with an outcome-free final ranking code path, not preregistration or a pristine prospective trial. | +| The selector has a declared stable rule. | A36: full budget, effective-PD feasibility, deterministic `B_u<=0.28`, then maximum expected objective; winner stable on `[0.259036,0.290491)`. | The endpoint cap appears chosen to force the midpoint. | Report the complete 3x3 grid, cap-change boundaries, and the independent December replay. | +| Selector stability is not selected-set validity. | December reselects `linear-005` before outcomes are opened; default `0.145650`, miscoverage `0.124925`. | The 90% conformal label is read as 90% funded-set coverage. | Lead with the observed miss: stable outcome-free selection does not transfer marginal conformal validity to optimizer-selected weights. | +| The selected funded set has an exact accounting audit. | Full OOT: return `$179,327.59`, default `0.039375`, miscoverage `0.036875`, endpoint `0.258051`, `B_u+V=0.294926`. | Accounting is confused with a coverage theorem. | The identity is deterministic after outcomes; it is not nominal selected-set coverage. | +| The Markov statement is secondary and conditional. | Threshold `0.574279`; tail-probability bound `0.316228` under weighted validity. | Bound is too loose or presented as a hard risk cap. | Use it as sensitivity only. Operational controls are `tau`, midpoint exposure, and observed funded-set metrics. | +| A40 is a matched point-PD comparison. | Same candidates, budget, concentration, LGD, solver, and `tau=0.17`. | Comparator changes multiple semantics or sees labels. | Only the risk score changes; neither optimization reads OOT outcomes. | +| Robustness has a measured price. | A40: `8.678%` realized-return cost and `7.9025` percentage-point default reduction. | Default and miscoverage reductions are conflated. | Report default reduction (`7.9025` pp) separately from miscoverage reduction (`0.5025` pp). | +| The midpoint is not the safest CRPTO policy. | A40: 75% blend return `$172,939.50`, default `0.035875`, threshold `0.516624`. | Selected point is sold as dominant. | It is the highest calibration expected-objective candidate under the declared screen. | +| Performance is temporally heterogeneous. | A37: CRPTO wins strongly in 2018H2; point PD dominates 2019H2 and 2020+. | Full-panel result is overgeneralized. | State the reversals in body and abstract; no universal dominance claim. | +| Funded-set composition is correctly labeled. | A38 uses letter grade recovered from `sub_grade` and stores conformal group separately. | Score-quantile groups are mistaken for loan grades. | Call A38 a business composition audit, not fairness certification. | +| Bootstrap uncertainty is bounded. | A39 primary month-cluster return interval `$163,421.14`--`$193,551.65`; funded-loan sensitivity `$162,706.17`--`$193,924.74`. | Interval is read as full model/selection uncertainty. | Both are fixed-allocation diagnostics; model, conformal recipe, selector, and optimizer remain fixed. | +| Earlier methods do not multiply the contribution. | Supplement A1--A34. | Paper reads as several papers or an uncontrolled tournament. | OCE/CVaR, SPO+, online-style checks, and external replications are diagnostics or context, not active selectors. | +| Reproducibility is substantive evidence quality. | Run tags, configs, A35--A40 builder, claim-sync tests, manifest validation. | Tooling is presented as the only novelty. | Lead with decision method and empirical implication; reproducibility makes them auditable. | -## Reviewer Objection Bank +## Do Not Claim -| Objection | Short response | -|---|---| -| "CP + RO already exists." | CRPTO instantiates the bridge for frozen credit PD artifacts, funded-set economics, exact audit, and reproducible governance. | -| "CP + RO is a direct combination, not a theory contribution." | The paper's theory is intentionally modest: Theorem 1 separates deterministic funded-set accounting from the Markov step under weighted funded-set validity. The contribution is the decision-certificate bridge: frozen PD artifact -> conformal endpoint -> robust funded set -> exact post-decision audit with reproducible governance. | -| "Adaptive selection breaks conformal validity." | Correct concern, and exactly why the paper isolates it: Assumption 1 states weighted funded-set validity explicitly, Theorem 1 separates the deterministic identity from the Markov step, and the exact audit checks the selected frozen funded set after the fact. | -| "This is one dataset." | Lending Club carries the certificate; Prosper/Freddie test transfer of the recipe without claiming new certificates. | -| "The AUC is not high enough." | The paper is not a credit-scoring leaderboard; calibrated probabilities are inputs to an auditable decision. | -| "Why distribute a binary instead of asking readers to retrain?" | The certified object is the frozen prediction-to-decision chain. E3/E4 show seed and temporal PD diagnostics are stable enough to support the binary choice, while the exact artifact hashes keep the submitted certificate fixed. | -| "The intervals are too wide to use." | Raw width is expected on a binary PD-scale interval; the paper evaluates whether upper endpoints rank downside risk and produce a funded set that passes Winkler, funded-set miscoverage, and exact alpha-safe checks. | -| "SPO+ has lower regret." | Correct; the paper reports a frontier where SPO+ buys regret and CRPTO buys verifiable risk controls; the pool93 frontier updates the funding certificate, not the SPO+ regret experiment. | -| "Why not live validation?" | Lending Club retail originations ended in 2020; prospective live validation is future protocol, not hidden current evidence. | +- pristine prospective or preregistered OOT evaluation; +- nominal conformal validity for optimizer-selected funded weights; +- universal dominance over point PD or decision-focused learning; +- causal return or default effects; +- legal fair-lending certification; +- live post-2020 Lending Club performance; +- that external Prosper/Freddie diagnostics replicate the final midpoint + selector exactly. + +## Required Headline Numbers + +- selected return: `$179,327.59`; +- selected default / miscoverage: `0.039375 / 0.036875`; +- `Gamma_CP / Gamma_residual`: `0.176102 / 0.088051`; +- endpoint / observed accounting / conditional threshold: + `0.258051 / 0.294926 / 0.574279`; +- point-PD return: `$196,369.14`; +- return cost / default reduction: `8.678% / 7.9025` pp; +- selector: `5/9` eligible, selected `tau=0.17`, `gamma=0.50`. +- deterministic selector cap / stability interval: `0.28 / [0.259036, 0.290491)`; +- December audit default / miscoverage: `0.145650 / 0.124925`; +- primary month-cluster return interval: `$163,421.14`--`$193,551.65`. diff --git a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md index 9b65f95..cf80bbd 100644 --- a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md +++ b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md @@ -1,85 +1,86 @@ # IJDS Cover Letter and Disclosure Draft -This file is for the editor-facing submission package. It is not part of the -double-anonymous reviewer packet unless the submission system explicitly asks -for the corresponding disclosure fields. +Editor-facing material only. Do not include it in the double-anonymous reviewer +packet unless ScholarOne requests the corresponding disclosure text. -## Cover Letter Core Paragraph +## Cover Letter Dear Editors, -We submit "CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit -Portfolio Decisions" for consideration at the INFORMS Journal on Data Science. -The paper studies credit allocation as data science for decisions rather than -as a predictive leaderboard. Its data component is a static Lending Club -out-of-time panel, supported by frozen Prosper and Freddie/Mendeley external -economic replications. Its method maps a frozen calibrated probability-of-default -artifact through Mondrian conformal intervals into a robust portfolio decision. -Its decision object is the funded set under a budget and risk cap, and its main -implication is an auditable model-risk surface: the promoted Lending Club body -point earns `$184.8K` on a `$1M` budget while passing an exact empirical -alpha-grid funded-set audit, and the declared pool93 finite-grid frontier -contains 50,010 deduplicated semantic policies with 27,508 all-alpha -above-floor policies. -An opt-in drift harness verifies that the prediction-to-decision certificate -chain regenerates bit-exactly from the frozen artifacts under the locked stack. -The contribution is intended for settings where decision auditability, -reproducibility, and model-risk governance matter as much as predictive rank. +We submit "CRPTO: A Calibration-Selected Conformal Guardrail for Credit +Portfolios" for consideration at the *INFORMS Journal on Data Science*. The +paper treats credit allocation as data science for decisions, +not as a credit-scoring leaderboard. A frozen calibrated PD model is combined +with an exactly replayed 90% Mondrian conformal endpoint. The resulting +midpoint score, `q=(p+u)/2`, constrains a `$1M` portfolio while point PD remains +in the expected-return objective. + +The final policy is selected from nine round-number candidates on November +2017 using a deterministic endpoint cap. Outcomes are stored separately from +its 12-column ranking frame, which contains no assumption-conditional +statistics. An outcome-free December replay selects the same rule; opening +outcomes afterward reveals +miscoverage `0.124925`, so the paper explicitly does not infer selected-set +validity from policy stability. On 276,869 out-of-time Lending Club loans, the +fixed policy earns `$179,327.59`, with weighted default `0.039375`. +A matched point-PD allocation earns `$196,369.14` with weighted default +`0.118400`. The paper reports both the `8.678%` return cost and the `7.9025` +percentage-point default reduction, together with temporal periods in which +the point-PD decision performs better. We therefore position CRPTO as an +auditable retrospective return-risk guardrail, not as a universal winner or +prospective deployment guarantee. + +The submission contributes an explicit prediction-to-decision contract, an +exact conformal replay, a temporally separated selector/audit, matched economic +comparisons, and a file-backed reproducibility package. These features align +with IJDS's emphasis on data, innovative methodology, decision relevance, and +reproducible evidence. + +Sincerely, + +[Author details supplied separately] ## Data and Code Availability -The submission body and supplement are double-anonymous. During review, the -manuscript refers to a reproducible companion package without exposing -author-identifying URLs. The submission will complete the IJDS Data and Code -Disclosure Form and acknowledge the accepted-paper reproducibility workflow. -After the venue permits disclosure, the companion can include: - -- public source code and Quarto manuscript sources; -- DVC metadata and pointers for processed artifacts and frozen model files; -- MLflow/DagsHub lineage for the CRPTO runs, subject to credential-free access - rules; -- raw-data source instructions from `RAW_DATA_SOURCE_NOTES.md` rather than - redistributed raw CSVs when source terms or file size make rehosting - inappropriate; -- Prosper and Freddie/Mendeley source notes for the external replication layer; -- the frozen extraction manifest and guardrail tests used to verify the - promoted frontier; -- the drift harness that recomputes the conformal interval and certificate - chain from frozen PD artifacts with zero endpoint drift under the locked stack; -- commands for regenerating paper tables, figures, HTML previews, and local - IJDS PDF verification drafts. - -No secrets, tokens, private DVC credentials, or local machine paths should be -included in the reviewer packet. - -If ScholarOne asks for the disclosure option in prose, the intended answer is: -code and manuscript sources will be released at acceptance; raw data are -public-source or source-controlled and therefore disclosed through source -instructions plus DVC pointers/processed artifacts when the journal workflow and -source terms permit. - -## Double-Anonymous Handling - -- Upload `paper/CRPTO_ijds.pdf` as the local body preview only if the official - template PDF is not yet required. -- Upload `paper/supplement_ijds.pdf` as the local supplement preview only if - the official workflow accepts HTML-print verification drafts. -- Keep public repository, DagsHub, MLflow, personal site, affiliation, and - author-identifying acknowledgements out of reviewer-facing files. -- Use this file or the submission system fields for disclosure timing, not the - anonymous manuscript body. - -## Editorial Fit - -The paper is positioned as data science for decisions: conformal prediction is -not only an uncertainty report, and robust optimization is not only a portfolio -heuristic. The central object is an executable, auditable decision recipe whose -numbers are backed by frozen artifacts, manifest regression tests, and -submission-ready tables and figures. - -The highest-risk interpretive boundary is also stated explicitly in the paper: -the Lending Club exact funded-set certificate is an exact accounting audit on -the frozen promoted portfolio under a declared weighted-validity assumption. The -Prosper and Freddie/Mendeley results are external economic replications and -exhaustiveness audits, not new exact funded-set certificates or prospective live -deployment guarantees. +The body and supplement are double-anonymous. During review they refer to a +reproducible companion without exposing author-identifying URLs. The IJDS Data +and Code Disclosure Form will state that, when venue policy permits, the +companion includes: + +- source code, configurations, and manuscript sources; +- A35--A40 evidence tables and active governance metadata; +- DVC metadata and artifact pointers for large processed data and model files; +- source instructions for Lending Club, Prosper, and Freddie/Mendeley data + rather than unauthorized redistribution; +- exact-alpha and calibration-selector replay commands; +- manifest, claim-sync, and publication-integrity tests; +- commands for regenerating tables, previews, and the official-template PDF. + +No secrets, tokens, private storage credentials, local usernames, or machine +paths belong in the reviewer package. + +Suggested ScholarOne prose: + +> Code, configurations, manuscript sources, and evidence-generation scripts +> will be released under the journal's accepted-paper reproducibility process. +> Public-source raw data are disclosed through source instructions; large or +> license-constrained artifacts are provided through documented pointers and +> integrity hashes where redistribution terms permit. + +## Anonymity Handling + +- Upload the `informs4` PDF built with `dblanonrev` as the manuscript. +- Upload the anonymous supplement separately. +- Keep the title page, affiliation, acknowledgements, repository ownership, + and personal URLs outside reviewer-facing files. +- Use this file and ScholarOne fields for editor-only disclosure timing. + +## Editorial Boundary + +The final ranking code is outcome-free with respect to OOT policy selection, +but earlier project development inspected the static OOT corpus. The manuscript +therefore says "retrospective lockbox replay," not "preregistered" or +"untouched holdout." Marginal/Mondrian coverage is not promoted to nominal +validity under optimizer-selected funded weights. OCE/CVaR, SPO+, online-style +checks, and external datasets remain diagnostics or context rather than +additional active methods. diff --git a/paper/submission/CRPTO_ijds_submission.tex b/paper/submission/CRPTO_ijds_submission.tex index 208fe8b..1696887 100644 --- a/paper/submission/CRPTO_ijds_submission.tex +++ b/paper/submission/CRPTO_ijds_submission.tex @@ -1,31 +1,20 @@ %% ===================================================================== %% CRPTO -- INFORMS Journal on Data Science (IJDS) submission manuscript %% ===================================================================== -%% Double-anonymous submission body in the official INFORMS class. The prose is -%% ported from the source of truth `paper/CRPTO_ijds.qmd`; keep the two in sync. +%% Double-anonymous official-template handoff synchronized with +%% paper/CRPTO_ijds.qmd. Build with latexmk, or use the Windows fallback: %% -%% REQUIRED (NOT vendored in this repo, per the no-vendoring rule): -%% - informs4.cls INFORMS document class -%% - informs2014.bst INFORMS BibTeX style -%% `informs4` is NOT on CTAN/TeX Live; download both from the INFORMS author -%% portal or Overleaf and drop them next to this file, then build: -%% https://pubsonline.informs.org/authorportal/latex-style-files -%% https://www.overleaf.com/latex/templates/template-for-informs-journal-on-data-science/sbthszxgycfn -%% latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex -%% If the local TinyTeX wrapper fails with runscript.tlu/nil, use: %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex %% bibtex CRPTO_ijds_submission %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex %% -%% `dblanonrev` keeps the manuscript anonymous (IJDS uses double-anonymous -%% review for submissions on/after 2025-01-01). Do NOT add author names, -%% affiliations, acknowledgements, or repository URLs to this body. +%% Pass 1 writes the .aux file, BibTeX writes the .bbl, pass 2 resolves the +%% bibliography and cross-references, and pass 3 stabilizes labels and pages. %% ===================================================================== \documentclass[ijds,dblanonrev]{informs4} -% --- Spacing / theorem / equation conventions (IJDS template defaults) --- -\usepackage{eqndefns-left} % flags display equations that exceed text width +\usepackage{eqndefns-left} \RequirePackage{tgtermes} \RequirePackage{newtxtext} \RequirePackage{newtxmath} @@ -35,9 +24,8 @@ \TheoremsNumberedThrough \EquationsNumberedThrough \ECRepeatTheorems -\MANUSCRIPTNO{} % left blank for initial submission +\MANUSCRIPTNO{} -% --- Math + graphics packages used by the manuscript --- \usepackage{amsmath,amssymb,mathtools} \usepackage{booktabs} \usepackage{graphicx} @@ -54,1055 +42,548 @@ \begin{document} \RUNAUTHOR{Anonymous} -\RUNTITLE{Conformal Robust Predict-Then-Optimize for Credit Portfolios} +\RUNTITLE{A Calibration-Selected Conformal Guardrail for Credit Portfolios} -\TITLE{CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit -Portfolio Decisions} +\TITLE{CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios} -% Authors hidden under dblanonrev; keep the block empty/anonymous. \ARTICLEAUTHORS{% \AUTHOR{} \AFF{} } \ABSTRACT{% -Credit allocation is a data-science-for-decisions problem: calibrated default -probabilities matter only after they shape which loans are funded. We introduce -Conformal Robust Predict-Then-Optimize (CRPTO), a post-hoc bridge that maps a -frozen calibrated probability-of-default model through Mondrian conformal -intervals into robust portfolio constraints and an empirical funded-set audit. -On a 276{,}869-loan out-of-time Lending Club evaluation, the selected policy -earns \$184.8K on a \$1M budget while passing the declared -eight-level alpha grid ($V(0.01)=0.035350$, -$\Gamma_{\mathrm{CP}}=0.162616$, Markov cap $0.345084$, zero violation). The -same exact frontier exposes conservative and economic endpoints, making the -return-bound trade-off visible rather than implicit. -Frozen external credit replications preserve the predeclared global conformal -gates and produce positive robust LP objectives, so the result is not confined -to the Lending Club panel. Across the four frozen external applications, the -price of robustness forms a positive premium series ordered by panel default -rate. The contribution is a -conformal-robust credit-portfolio decision certificate with a distribution-free -Markov bound under weighted funded-set validity: it connects real credit data, -calibrated predictive models, robust funding decisions, and a validation harness -that rebuilds the prediction-to-decision chain from frozen inputs -while keeping the statistical guarantee boundary explicit.% +Credit models matter only through the decisions they change. We study how +finite-sample predictive uncertainty can constrain a loan portfolio after a +probability-of-default (PD) model has been frozen. Conformal Robust +Predict-Then-Optimize (CRPTO) recomputes a 90\% Mondrian conformal upper endpoint +exactly, forms the transparent decision score $q_i=(p_i+u_i)/2$, and places +$q_i$ in a portfolio-risk constraint while retaining point PD in the economic +objective. Nine round-number policies are ranked on November 2017 without +default, realized-return, miscoverage, or assumption-conditional columns; an +outcome-free December replay selects the same rule before its outcomes are +opened. That audit misses nominal funded-set coverage, making the statistical +boundary observable rather than rhetorical. The policy is then replayed on +276{,}869 out-of-time Lending Club loans. +It funds 308 loans and earns \$179{,}327.59 on a \$1M budget, with weighted +default 0.039375, weighted miscoverage 0.036875, and conformal endpoint budget +0.258051. A matched point-PD allocation earns \$196{,}369.14 but has weighted +default 0.118400 and endpoint budget 0.921317. Thus CRPTO pays 8.678\% of +realized return for a 7.9025 percentage-point default reduction; the advantage +reverses in some temporal slices, so we do not claim universal dominance. The +contribution is a small, auditable prediction-to-decision guardrail with an +exact replay, a deterministic selector, and explicit statistical boundaries.% } -\KEYWORDS{conformal prediction; robust optimization; predict-then-optimize; -credit risk; portfolio optimization; reproducible data science} +\KEYWORDS{conformal prediction; predict-then-optimize; credit risk; portfolio +optimization; calibration; reproducible data science} \maketitle -% ===================================================================== \section{Introduction}\label{sec:intro} -Credit allocation is a predict-then-decide problem. A lender first estimates a -probability of default (PD), then chooses which loans to fund under a budget and -risk appetite. The modeling literature has become very good at the first step: -calibration, discrimination, and backtesting are now standard ingredients of -credit-risk model validation \citep{lessmann2015,chen2024creditrisk}. The second step is -less settled. Once a calibrated PD enters an optimizer, uncertainty is often -treated as a reporting diagnostic rather than as a constraint that can change the -funded set. - -That separation is uncomfortable in auditable credit decisions. A portfolio policy -can have a reasonable average PD and still concentrate probability mass in loans -where the model is most uncertain. Conversely, a policy that is too conservative -can pass every risk check while destroying economic value. The scientific question -in this paper is therefore not whether one can build a slightly better credit -classifier. It is whether finite-sample predictive uncertainty can be carried into -a robust portfolio decision in a way that is transparent enough for a reviewer to -audit. This has practical stakes. In a pre-registered randomized trial, -conformal prediction sets improved human decision making relative to fixed-size -sets with the same coverage \citep{cresswell2024}. CRPTO takes that -committee-facing idea into a credit portfolio setting, where the uncertainty -summary must change a funding decision or it is just another report. - -CRPTO answers this question with a post-hoc, reproducible pipeline. It starts from -a calibrated CatBoost PD model, constructs Mondrian conformal intervals over -PD-scale predictions, and maps the upper conformal endpoint into robust portfolio -constraints. The pipeline is modular by design: the predictive model, conformal -layer, optimization policy, and paper outputs each have separate contracts. -That separation lets the paper ask whether a frozen prediction system can be -converted into a defendable decision system without reopening hyperparameter -search whenever the manuscript or appendix changes. - -The empirical setting is the Lending Club retail-loan panel, with an out-of-time -evaluation set of 276{,}869 loans. The consolidated frontier contains 50{,}010 -deduplicated semantic policies, of which 27{,}508 pass every declared alpha -level and exceed the return floor. From that declared finite frontier, the -selected policy is the highest-return point that passes the eight -alpha levels under the declared $0.345$ Markov cap; it is neither a continuous -global optimum nor the economic endpoint. The selected point earns \$184.8K on -a \$1M budget and passes the exact empirical funded-set audit at -$\alpha=0.01$. The headline result is not a single lucky allocation. It is a -reproducible bridge from calibrated probabilistic learning to robust, auditable -credit portfolio choice, with the return-bound frontier reported rather than -hidden behind one selected point. - -To address the natural single-dataset concern without reopening the Lending Club -champion, the paper also freezes two external economic replications: Prosper -final-status marketplace loans and a Freddie/Mendeley single-family mortgage -panel with out-of-sample and out-of-time splits. These replications are not new -champions; they test whether the same PD-to-conformal-to-LP recipe remains -economically usable on different credit products. - -The paper makes five contributions. First, it gives a CRPTO construction for -credit portfolios: frozen calibrated PD, Mondrian conformal uncertainty, and -robust budgeted optimization as a post-hoc decision audit. Second, it proves a -distribution-free Markov bound under weighted funded-set validity -(Theorem~\ref{thm:funded-set-bound}) -that splits realized portfolio loss into the conformal upper-endpoint budget -$B_u(\alpha)=\tau+(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ and the weighted -miscoverage $V(\alpha)$, with supplement propositions showing that Markov is -optimal under the stated assumption (A.1) and locating the cluster structure -that would tighten it (A.2). Third, it locates that construction relative to -data-driven robust optimization, P2P lending portfolio models, conformal credit -scoring, and decision-focused learning. Fourth, it provides an evidence-backed -empirical study where every table and figure is generated from frozen outputs -rather than manually transcribed summaries. Fifth, it uses a three-level -evidence ladder: the Lending Club funded set carries the certificate, Prosper -and Freddie/Mendeley test recipe transfer as external economic replications, -and the tail, dependence, online, and robustness appendices stay diagnostic. -The key claim is narrow: CRPTO maps frozen calibrated -PD models into a robust funded set, reports the portfolio-level conformal -premium $\Gamma_{\mathrm{CP}}$, and verifies exact alpha-safe weighted -miscoverage on the promoted Lending Club portfolio and its surrounding finite-grid -frontier. The same conformal and LP gates remain viable on two additional -credit datasets, while the paper keeps the boundary between paper revision and -new model-search evidence visible. - -Read as data science for decisions, the paper's four components are explicit: -the data are a static Lending Club OOT panel plus Prosper/Freddie external stress -tests; the model is the CRPTO bridge from calibrated PD to Mondrian conformal -uncertainty to robust LP; the decision is the funded set under budget and risk -caps; and the implication is a reproducible audit surface for model-risk -committees. +Credit allocation is a contextual optimization problem. A lender estimates a +probability of default (PD) and then chooses which loans to fund under capital, +concentration, and risk constraints \citep{sadana2025contextual}. Credit-scoring +research has strengthened the first step through discrimination benchmarks, +probability calibration, and cost-aware evaluation +\citep{lessmann2015,chen2024creditrisk,yang2025costaware}. Yet a calibrated +probability does not specify how much model uncertainty a portfolio should +bear. A point-PD optimizer can concentrate capital in loans that appear +attractive where the predictive model is least certain. + +Conformal prediction offers finite-sample coverage language under explicit +exchangeability conditions \citep{vovk2005,angelopoulos2023}, while robust +optimization makes uncertainty operational through feasibility sets +\citep{bertsimas2004,goldfarb2003robustportfolio}. Joining them is attractive, +but an applied decision paper still has to answer three questions. Which +conformal level is informative rather than nearly vacuous? How is a policy +selected without using the outcomes on which it is later reported? What +economic value is lost when conformal uncertainty actually changes the funded +set? + +CRPTO answers those questions with one deliberately simple policy. A frozen, +calibrated CatBoost model produces $p_i$. An exactly replayed 90\% Mondrian +recipe produces upper endpoint $u_i$. The portfolio uses their midpoint, +\begin{equation}\label{eq:midpoint} +q_i=p_i+0.5(u_i-p_i)=\frac{p_i+u_i}{2}, +\end{equation} +inside the risk constraint. The economic objective remains point-PD expected +net return. This separation is substantive: $p_i$ prices expected loss, while +$q_i$ limits the uncertainty the funded portfolio may carry. It also removes +nonlinear caps, tail rules, and uncertainty penalties that made an earlier +research frontier difficult to explain. + +The empirical design uses a temporal Lending Club panel. The conformal recipe +is fit inside the calibration period, November 2017 ranks a declared +$3\times3$ grid, and December replays the outcome-free selector before auditing +the fixed decision. The ranking artifact contains no defaults, realized +returns, miscoverage, or assumption-conditional statistics. The fixed rule is +then evaluated on January 2018 through September 2020 originations. Earlier +project development inspected this static OOT corpus, so the final run is a +transparent retrospective lockbox replay, not a pristine prospective or +preregistered trial. + +The paper makes three contributions. First, it gives an auditable +prediction-to-decision construction in which the economic objective and the +conformal guardrail have separate roles. Second, it replaces approximate +cross-alpha scaling with an exact replay, a deterministic endpoint screen, and +a temporally separated selector audit. Third, it reports the price and limits +of that guardrail against matched point-PD and more-conservative comparators, +including a pre-OOT funded-set coverage miss and temporal slices where CRPTO +wins and slices where it does not. The novelty is the closed, inspectable +decision protocol for a frozen credit model, not a claim that conformal +prediction, robust optimization, or credit scoring is individually new. \begin{figure}[t] - \centering - \includegraphics[width=\textwidth]{crpto_fig1_journal_pipeline.png} - \caption{CRPTO maps frozen calibrated PD models through a Mondrian conformal - uncertainty layer into an auditable robust portfolio decision and exact - funded-set certificate.} - \label{fig:pipeline} +\centering +\includegraphics[width=0.92\textwidth]{crpto_fig1_journal_pipeline.png} +\caption{CRPTO carries a frozen calibrated PD through an exact conformal +replay, a simple portfolio guardrail, and a funded-set audit.} +\label{fig:pipeline} \end{figure} -% ===================================================================== \section{Related Work}\label{sec:related} -CRPTO builds on conformal prediction, especially split conformal methods and their -risk-control extensions \citep{vovk2005,angelopoulos2023,angelopoulos2024foundations,angelopoulos2024risk,bates2021rcps}. -The relevant property is not that conformal intervals are the narrowest possible -uncertainty summaries. It is that they provide distribution-free coverage language -under explicit exchangeability assumptions, and that this language can be audited -without trusting a fully parametric posterior. Mondrian and group-conditional -variants are especially natural in credit because risk grades are already used as -business and governance partitions \citep{bostrom2021,gibbs2024,zhou2024}. - -The second foundation is robust optimization. Classical robust optimization frames -uncertainty as a set against which a decision must remain feasible, with the price -of robustness made visible as a design trade-off \citep{bertsimas2004}. Robust -portfolio selection makes that trade-off operational for allocation under -parameter uncertainty \citep{goldfarb2003robustportfolio}, whereas -distributionally robust optimization broadens the uncertainty object toward moment -or ambiguity sets \citep{delage2010dro}. Recent work connects conformal prediction -and robust optimization more directly by using conformal uncertainty sets in -downstream decisions \citep{johnstone2021,patel2024,sun2024ptc,hu2026crc}. That -line certifies the uncertainty set \emph{before} the decision: coverage of the -conformal region is the guarantee, and the downstream decision inherits it. -CRPTO follows this line but audits the other side of the decision as well: -after the optimizer selects a funded set, the realized weighted miscoverage -$V(\alpha)$ and the deterministic budget identity of -Theorem~\ref{thm:funded-set-bound} -are evaluated exactly on that funded set, so the certificate concerns the -allocation actually purchased rather than the input set alone. Its empirical -emphasis also differs: the uncertainty set is -not an abstract benchmark instance, but a credit-risk interval model with -auditable lineage, paper tables, and model-risk documentation. - -The third foundation is predict-then-optimize and decision-focused learning. SPO+ -and modern decision-focused learning ask models to respect the loss surface induced -by the downstream decision \citep{elmachtoub2022,donti2017,mandi2024}. IJDS work on -causal decision making sharpens the same warning: once an estimate feeds an -action, the relevant target can become the assignment rule rather than only the -intermediate effect-size estimate \citep{fernandezloria2022causaldecision}. CRPTO is more -conservative. It does not retrain the PD model end-to-end through the optimizer. -Instead, it asks what can be achieved when a calibrated predictive system is -already frozen and the decision layer must remain explainable to credit-risk -reviewers. Robust losses for decision-focused learning \citep{schutte2024robust} -share this protective intent, but operate at training time rather than as a -post-hoc auditable constraint. - -The fourth foundation is machine learning for credit scoring, where gradient -boosting and benchmark studies define the performance frontier on retail panels -\citep{lessmann2015,ayari2026,boosting2025default,xia2017}. Recent IJDS -credit-risk work shows how richer data structures such as firm graphs can improve -rating prediction \citep{das2023creditgraph}, and cost-aware calibration work makes -explicit why probability quality matters when predictions feed asymmetric -downstream decisions \citep{yang2025costaware}. Work specific to the Lending Club -platform spans alternative-data fintech lending \citep{jagtiani2019altdata}, -scorecard equity \citep{albanesi2024credit}, two-stage learning under fragmentary -data \citep{zheng2026twostage}, P2P portfolio selection and profit scoring -\citep{zhao2016p2pportfolio,serrano2016profitscoring}, and operations-research -treatments of the same investment-decision problem CRPTO studies -\citep{aior2025lendingclub}. CRPTO does not compete on raw ranking against this -literature; its champion AUC is mid-range. The contribution is the -auditable bridge from a calibrated, frozen PD model to a robust portfolio decision, -not another point on the credit-scoring leaderboard. - -Finally, recent work on conformal model selection for robust optimization, -multi-distribution conformal validity, online conformal portfolio methods, -end-to-end conformal risk training, and conformal satisficing -\citep{bao2025croms,yang2026multidistribution,liu2026portfolio,yeh2025training,zhao2025robust} -motivates the journal-strengthening package in this paper. We use those ideas where -they can be evaluated from the frozen CRPTO evidence: OCE/CVaR -\citep{rockafellar2000cvar,bental2007oce} appears as a tail-risk diagnostic, robust -satisficing appears as committee-style margin evidence, and SPO+ motivates the -regret-auditability frontier. The method-changing variants---optimized OCE/CVaR -objectives, online or multi-dataset protocols, causal layers, and hybrid -decision-focused training---remain future work rather than hidden acceptance -criteria. - -\FloatBarrier -\subsection{Closest Work Boundary}\label{sec:closest} - -CRPTO is not first in any broad individual family; its claim is the combination of -calibrated PD, conformal uncertainty, robust credit-portfolio optimization, exact -funded-set validation, and reproducible governance. - -\begin{table}[h] - \centering - \small - \caption{Closest work boundary for CRPTO.} - \label{tab:closest-work} - \begin{tabular}{p{0.23\textwidth}p{0.23\textwidth}p{0.23\textwidth}p{0.23\textwidth}} - \toprule - Neighboring literature & What it contributes & What CRPTO adds & Why not enough \\ - \midrule - P2P/Lending Club OR & Robust or multi-objective funding models. & Conformal PD uncertainty plus exact funded-set validation. & Leaves a gap between prediction uncertainty and a post-allocation certificate. \\ - Conformal credit scoring & Intervals for score uncertainty. & A downstream robust portfolio decision. & Does not audit a budgeted funded set or economic policy. \\ - Conformal robust optimization & Conformal sets in robust decisions. & Frozen credit-risk lineage, PD governance and funded-set certificate. & Not credit-specific enough for PD lineage. \\ - Decision-focused learning & Regret-aligned training. & Post-hoc governance for existing calibrated PD models. & Does not certify risk controls after a frozen PD model. \\ - Conformal finance portfolios & Financial portfolio applications. & Retail-credit payoffs, default risk and model-risk documentation. & Market-return portfolios face different payoffs and controls. \\ - \bottomrule - \end{tabular} -\end{table} - -The novelty is the closed loop, not any single row of the table: a frozen calibrated -credit model becomes conformal upper endpoints, those endpoints enter a robust -funded-set decision, and the purchased allocation is audited after the decision with -file-backed governance. None of the neighboring lines jointly supplies that -frozen-model-to-funded-set certificate for a credit portfolio. - -% ===================================================================== -\section{Method}\label{sec:method} - -\subsection{Calibrated PD Layer}\label{sec:method-pd} - -The predictive layer estimates a one-period default probability for each loan. The -champion model is a CatBoost classifier trained on the frozen feature contract and -calibrated before it is exposed to conformal and optimization layers. The paper -reports discrimination and probability quality together: the PD layer reaches - AUC $0.7139$, Brier score $0.1544$, and expected calibration error approximately - $0.0070$ on the paper-facing evaluation summary. These numbers matter because the -optimizer consumes probabilities, not rankings alone. - -Calibration is treated as a contract. The downstream layers do not receive a -free-form classifier; they receive a calibrated PD vector, feature metadata, and a -model contract that fixes feature order and categorical handling. This prevents a -common reproducibility failure in applied predict-then-optimize studies: a table can -change because a preprocessing file changed, even though the optimization code -did not change. - -\subsection{Mondrian Conformal Layer}\label{sec:method-cp} - -For each loan $i$, let $\hat{p}_i$ denote the calibrated PD. The conformal layer -forms prediction intervals on the PD scale and records the upper endpoint -$u_i(\alpha)$. Operationally, CRPTO evaluates Mondrian partitions rather than a -single global interval. The selected uncertainty layer uses a score-decile-based -Mondrian partition selected by out-of-time interval quality, while grade-based -partitions remain the natural governance baseline. The score-decile choice keeps -each cell well populated even at the tight $\alpha=0.01$, where a per-cell $99\%$ -upper endpoint needs on the order of $100$ calibration points; finer grade-period -partitions are sparser, and the supplement flags where small cells weaken coverage -(notably the external Freddie panel). - -The resulting conformal summary is more than a scalar coverage number. -The paper-facing metrics include 90\% coverage $0.9297$, 95\% coverage $0.9664$, -average 90\% interval width $0.7842$, minimum group 90\% coverage $0.9190$, and 90\% - Winkler score $1.1107$ for the promoted conformal winner. Because the outcome is - binary and intervals live on the clipped PD scale, raw width is not a standalone - utility claim. Its decision role is relative: upper endpoints rank loans by - protected downside risk, the funded-set audit shows mean upper endpoints rising - from $0.12529$ in A--B to $0.52587$ in E--G, and the promotion gate uses Winkler - score, funded-set miscoverage, and $\Gamma_{\mathrm{CP}}$ rather than treating - narrowness as the objective. The full gate also scores - material coverage, group coverage, interval width, and alert rate, but not - exact-nominal-coverage backtests: conformal intervals over-cover by design, so - testing for exact nominal coverage would penalize the safety margin the method is - meant to provide on a large out-of-time sample. - -\subsection{Robust Portfolio Layer}\label{sec:method-lp} - -The decision variable $x_i$ is the allocation fraction for each eligible loan; -$x_i a_i$ is the funded exposure. The optimizer maximizes expected net economic -return under a \$1M budget and policy constraints that cap portfolio risk after -replacing point PD estimates with conformal upper endpoints. The selected body -point has $\texttt{risk\_tolerance}=0.1715$, -$\texttt{policy\_mode}=\texttt{capped\_blended\_uncertainty}$, policy parameter -$\gamma=0.5475$, and $\texttt{uncertainty\_aversion}=0.05$. - -Schematically, the robust decision layer solves -\[ -\begin{aligned} -\max_x\quad & \sum_i x_i a_i \left(c_i - \tilde p_i(\alpha,\gamma)\,L\right) \\ -\text{s.t.}\quad & \sum_i x_i a_i \le B,\\ -& \sum_i x_i a_i \tilde p_i(\alpha,\gamma) - \le \tau \sum_i x_i a_i,\\ -& 0 \le x_i \le \bar x_i, -\end{aligned} -\] -where $a_i$ is exposure, $c_i$ is the loan coupon, $L$ is the loss-given-default -($L=0.45$ in the frozen evaluation), $\tau$ is the risk-tolerance cap, and -\[ -\tilde p_i(\alpha,\gamma)=\hat p_i+\gamma\left(u_i(\alpha)-\hat p_i\right) -\] -on the PD scale, clipped to the feasible probability range. The objective is the -\emph{expected} net return $c_i-\tilde p_i L$; the headline realized return is -the post-hoc accounting of the same funded set on observed -defaults (a funded loan earns $c_i a_i$ if it survives and loses $L a_i$ if it -defaults). Separating the optimized expectation from the realized accounting is -deliberate: the optimizer never sees outcomes, so the realized figure is an -out-of-sample audit of the policy, not the objective it maximized. Additional -operational filters and caps live in the frozen policy configuration; the -manuscript displays the core statistical-to-decision contract because that is -the reusable CRPTO pattern. - -Two quantities are kept separate throughout the paper. The lowercase $\gamma$ is a -policy parameter controlling how the optimizer blends uncertainty in the portfolio -rule. $\Gamma_{\mathrm{CP}}$, by contrast, is a portfolio-level conformal metric -computed after the funded set is chosen: -\[ -\Gamma_{\mathrm{CP}}(\alpha)=\sum_i w_i\left(u_i(\alpha)-\hat p_i\right)_+. -\] -The selected body point has - $\Gamma_{\mathrm{CP}}(\alpha=0.01)=0.162616$, weighted miscoverage - $V(\alpha=0.01)=0.035350$, and zero exact violation at $\alpha=0.01$. This distinction -is small typographically but central for auditability. - -% ===================================================================== -\section{Theory}\label{sec:theory} - -The theoretical role of conformal prediction in CRPTO is modest and explicit. For a -fixed allocation evaluated on exchangeable calibration/test data, conformal coverage -controls the expected rate at which outcomes fall outside the constructed uncertainty -intervals. When funded-set weights are non-negative and normalized, this yields a -weighted miscoverage quantity $V(\alpha)$ that can be monitored after the decision. -Two statements are kept separate. The deterministic portfolio identity -(Theorem~\ref{thm:funded-set-bound}(i)) holds for any allocation and needs no -distributional assumption; it is the accounting the exact certificate verifies. The -probabilistic statement (Theorem~\ref{thm:funded-set-bound}(ii)) is a Markov argument -that requires weighted funded-set validity, $\mathbb{E}[V(\alpha)]\le\alpha$, stated as -Assumption~\ref{asm:weighted-validity}. This is a modeling assumption, not a property -the single frozen draw establishes: on the selected funded set the realized weighted -miscoverage is $V(0.01)=0.035350$, \emph{above} the nominal $\alpha=0.01$, the expected -price of evaluating an adaptively selected subportfolio rather than a fresh population. -What the paper certifies is therefore the exact accounting together with the safety -level $V\le\sqrt{\alpha}$ that Markov delivers, not a claim that the funded set attains -nominal $\alpha$-coverage or that post-selection evaluation creates a stronger -conformal guarantee. - -\begin{figure}[t] - \centering - \includegraphics[width=0.95\textwidth]{crpto_fig20_bound_claim_layers.pdf} - \caption{The CRPTO bound claim stack separates conformal endpoints, - deterministic portfolio accounting, the weighted-validity assumption, and the - frozen exact certificate.} - \label{fig:bound-stack} -\end{figure} +CRPTO sits at the intersection of conformal prediction, robust optimization, +and decision-focused learning. Split conformal methods provide marginal +coverage without a parametric posterior, while Mondrian variants condition on +declared partitions \citep{vovk2005,bostrom2021}. Conditional and weighted +extensions require additional structure, and exact conditional coverage is +generally unavailable without restrictive assumptions +\citep{barber2021limits,barber2023beyond,jonkers2024wcps}. We therefore +distinguish population or partition coverage from coverage after a portfolio +has adaptively reweighted the loans. + +Data-driven robust and contextual optimization translate predictive +uncertainty into decisions +\citep{bertsimas2018datadriven,bertsimas2020prescriptive,sadana2025contextual}. +Recent work uses conformal sets in robust optimization +\citep{johnstone2021,patel2024,sun2024ptc,hu2026crc}. CRPTO is an applied +complement: it retains a frozen credit PD model, exposes the funded rows and +uncertainty premium, and compares the resulting allocation with a matched +point-PD decision. + +Decision-calibrated prediction sets and inverse conformal risk control go +further by calibrating downstream violation or regret and, after robustness +selection, using a separate split to restore risk-estimation validity +\citep{zhou2026creme,stratigakos2026decision_calibrated_sets}. CRPTO does not +import that guarantee into a batch portfolio where the optimizer chooses the +funded weights. It separates a deterministic endpoint screen from an +independent post-selection audit and reports when funded-set miscoverage misses +its nominal target. + +Decision-focused learning and SPO+ train predictions against downstream regret +\citep{donti2017,elmachtoub2022,mandi2024}. That is a different institutional +choice. CRPTO asks what can be done after a calibrated model already exists and +must remain frozen for governance. A synthetic SPO+ diagnostic remains in the +online supplement; it is not mixed with the real-dollar portfolio results. + +Credit-allocation research already covers profit scoring, P2P investment +recommendation, robust loan portfolios, rejection, and multiobjective risk +\citep{guo2016p2p,zhao2016p2pportfolio,serrano2016profitscoring,chi2019p2p, +babaei2020p2p,xu2025profit_uncertainty_credit,xu2024profit_risk_credit}. +Conformal credit scoring also means that the safe claim is not first use of +conformal prediction in credit \citep{kawasumi2026ordinal}. The remaining gap +is a file-backed protocol that shows exactly how a conformal endpoint changes a +budgeted credit decision and what that change costs. -The bound is read as three linked objects: a deterministic accounting identity, -an explicitly stated statistical assumption, and the frozen empirical certificate -that verifies the promoted decision exactly on the 276{,}869-loan OOT evaluation. -We now state the first two formally. Fix the promoted allocation $x$, chosen -without access to OOT labels, with exposures $a_i>0$ and funded-set weights -$w_i=x_i a_i/\sum_j x_j a_j$, so that $w_i\ge 0$ and $\sum_i w_i=1$. For each -funded loan let $Y_i\in[0,1]$ be the realized outcome on the PD scale, -$u_i(\alpha)\in[0,1]$ its upper conformal endpoint, and -$Z_i(\alpha)=\mathbf{1}\{Y_i>u_i(\alpha)\}$ the miscoverage indicator. The -weighted funded-set miscoverage is $V(\alpha)=\sum_i w_i Z_i(\alpha)$. -Probabilities and expectations below are taken over the exchangeable -calibration/test draw, conditional on the frozen recipe, declared partitions, -and allocation rule. - -\begin{assumption}[Weighted funded-set validity]\label{asm:weighted-validity} -$\mathbb{E}[V(\alpha)]\le\alpha$ under the funded-set weights $w$ for that draw; -in words, the expected weighted miss rate on the funded dollars is no larger -than the nominal conformal level. -\end{assumption} - -Assumption~\ref{asm:weighted-validity} is the explicit price of evaluating a selected -portfolio rather than a single population, and it does not follow from marginal split -conformal. The funded-set weights $w_i\propto x_i a_i$ are chosen by the optimizer and -depend on the conformal endpoints $u_i(\alpha)$, so they are not measurable with respect -to the Mondrian partition and inherit no per-cell coverage guarantee. The assumption is -therefore stated, audited empirically after the frozen selection, and never silently -upgraded to a guarantee---and the audit does not find it slack: the realized -$V(0.01)=0.035350$ exceeds $\alpha=0.01$, so the operative safety level is the weaker -$V\le\sqrt{\alpha}$. - -\begin{theorem}[Distribution-free funded-set risk bound]\label{thm:funded-set-bound} -Let $B_u(\alpha)=\sum_i w_i u_i(\alpha)$ be the weighted conformal -upper-endpoint budget of the funded set. Then -(i) deterministically, $\sum_i w_i Y_i \le B_u(\alpha) + V(\alpha)$; and -(ii) under Assumption~\ref{asm:weighted-validity}, for every $t>0$, -$\mathbb{P}(V(\alpha)\ge t)\le\alpha/t$, and in particular -\[ -\mathbb{P}\Bigl(\textstyle\sum_i w_i Y_i \;\ge\; B_u(\alpha)+\sqrt{\alpha}\Bigr) - \;\le\; \sqrt{\alpha}. -\] -\end{theorem} - -\proof{Proof sketch.} -Since $Y_i\le u_i(\alpha)+Z_i(\alpha)$ for every loan, part~(i) follows by -taking the $w$-weighted sum; it is portfolio accounting and needs no -probability. Part~(ii) is Markov's inequality applied to the nonnegative -variable $V(\alpha)$ with $\mathbb{E}[V(\alpha)]\le\alpha$ \citep{ghosh2002}, -combined with~(i). The full proof is in Online Supplement Appendix~A. \Halmos -\endproof - -\begin{remark}[The optimizer's cap versus the endpoint budget]\label{rem:endpoint-budget} -The robust layer does not constrain $B_u(\alpha)$ directly; it caps the -$\gamma$-blended PD, $\sum_i w_i \tilde p_i(\alpha,\gamma)\le\tau$, with -$\tilde p_i=\hat p_i+\gamma(u_i(\alpha)-\hat p_i)$ and $\gamma\in[0,1]$. Because -$\Gamma_{\mathrm{CP}}(\alpha)=\sum_i w_i(u_i(\alpha)-\hat p_i)$, the endpoint -budget decomposes exactly as -\[ -B_u(\alpha)=\sum_i w_i \tilde p_i(\alpha,\gamma)+(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha) - \;\le\; \tau+(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha), -\] -with equality when the cap binds. The term $(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ -is the conformal robustness premium the optimizer leaves un-internalized at -$\gamma<1$. For the selected body point ($\tau=0.1715$, $\gamma=0.5475$, binding cap), -$B_u(0.01)\le 0.1715+0.4525\,(0.162616)=0.245084$, so the deterministic -accounting bound reads -$\sum_i w_i Y_i\le 0.245084+V(0.01)=0.280434$. The exact audit reports zero -deterministic violation; the conservative Markov cap for the same point is -$0.345084$. -\end{remark} - -\begin{remark}[Why $t=\sqrt{\alpha}$, and why Markov]\label{rem:sqrt-alpha} -The choice $t=\sqrt{\alpha}$ is made for interpretability, not optimality: it -reads cleanly as ``miscoverage exceeds $\sqrt{\alpha}$ with probability at most -$\sqrt{\alpha}$'' (for $\alpha=0.01$, a $0.10$ excess with probability at most -$0.10$). Markov is deliberately the weakest defensible argument: it uses only -the first moment. Supplement Propositions~A.1--A.2 separate the boundary: -under Assumption~1 alone the best second-moment (Cantelli) threshold is -\emph{worse} than Markov, while explicit cross-cluster structure is the extra -condition under which Hoeffding-style tightening becomes available -\citep{hoeffding1963,boucheron2013concentration}. Those tightenings are kept -in the online supplement (A21) rather than the body, because the contribution -is the auditable decision construction, not the sharpest tail bound. The exact -certificate in this paper is the empirical audit of the frozen selected policy, -not a stronger post-selection conformal theorem. -\end{remark} - -The theorem and the two supplement propositions should be read as one small -triptych. Theorem~\ref{thm:funded-set-bound} gives the paper's guarantee once -weighted funded-set validity is accepted. Supplement Proposition~A.1 shows -that, without additional structure, Markov is not a placeholder for a missing -second-moment bound; it is the sharp first-moment statement. Supplement -Proposition~A.2 then asks what extra structure would buy a tighter threshold. -In this temporal credit panel the defensible version is cross-period or -period-grade independence after the frozen recipe and allocation are fixed: -within a period, grade, or period-grade cell, defaults and interval misses may -remain dependent. The observed funded set is too exposure concentrated for that -cluster argument to tighten the headline bound, which is why the body keeps -Markov and the supplement reports the cluster calculation as a sensitivity -check. - -% ===================================================================== -\section{Experimental Design}\label{sec:design} - -The empirical study uses Lending Club retail-loan data covering originations from -2007 through 2020. The raw panel is cleaned into a static feature store and split -temporally with a January 2018 cutoff, so calibration and evaluation use only loans -originated after the training window. The calibration block plays the dual role -required by split conformal: held out from training and used only to fit the -conformal quantiles. The final evaluation set contains 276{,}869 loans, large enough -to stress both probability calibration and decision-level robustness. - -\begin{table}[h!] - \centering - \caption{Temporal out-of-time split. The January 2018 cutoff keeps the test - window (including the 2020 COVID regime) strictly after training and calibration. - Monthly vintage labels are shown for readability; split assignment is - row-disjoint in code, so the shared March 2017 label marks the internal cutoff - rather than duplicated loans.} - \label{tab:split} - \resizebox{\textwidth}{!}{% - \begin{tabular}{llrl} - \toprule - Split & Period & Loans & Role \\ - \midrule - Train & Jun 2007 -- Mar 2017 & 1{,}346{,}311 & Fit PD model and monotonic constraints. \\ - Calibration & Mar 2017 -- Dec 2017 & 237{,}584 & Fit Mondrian conformal quantiles (held out). \\ - Test (OOT) & Jan 2018 -- Sep 2020 & 276{,}869 & Coverage, portfolio decision, exact funded-set audit. \\ - \bottomrule - \end{tabular} - }% +\begin{table}[t] +\centering +\small +\caption{Closest-work boundary for CRPTO.} +\label{tab:closest} +\begin{tabular}{p{0.20\textwidth}p{0.34\textwidth}p{0.37\textwidth}} +\toprule +Family & Existing contribution & CRPTO boundary \\ +\midrule +Credit scoring & Calibrated and cost-aware PD & PD is an input contract; no AUC-leadership claim. \\ +Robust credit portfolios & Economic selection under uncertainty & Exact conformal endpoint becomes a simple guardrail. \\ +Conformal robust optimization & Coverage-backed uncertainty sets & Frozen credit stack, funded-set audit, matched economics. \\ +Decision-focused learning & Training-time regret reduction & Frozen predictor and post-hoc auditability. \\ +Decision-risk calibration and valid selection & Calibrated decision losses or validity after set selection & Deterministic selector plus diagnostic audit; selected-set validity is not claimed. \\ +\bottomrule +\end{tabular} \end{table} -The out-of-time design is adversarial: the test window spans an -expansion (2018--2019) and a regime break (2020 COVID), so coverage and the -funded-set certificate are measured under a documented distribution shift rather -than on a random split that would let the model see the future. - -The design distinguishes three kinds of computation. Predictive and conformal -searches choose models, calibration, partitions, and policy families. Those -searches are frozen for this manuscript. Paper-facing reruns regenerate tables, -figures, evidence summaries, and manuscript surfaces from frozen inputs. This -separation is central to the reproducibility claim: the manuscript can evolve -without quietly reopening the policy search that selected the promoted result. -Governance keeps the two certified results distinguishable: the frozen upstream -record defines the declared return floor, while the selected-policy governance -files define the body point and its frontier. Automated claim-sync tests assert -that every body-claim number printed in the manuscript and supplement matches -those files. A validation harness additionally rebuilds the promoted conformal -intervals from the frozen PD binaries and recorded recipe and verifies the -published endpoints and coverage summaries. Gradient-boosted retraining is not -presented as the routine reproduction target across machines; retraining would -be a new research run, not the routine reproduction path for this submission. - -All primary evidence objects are represented as files with explicit ownership: model -binaries, calibration objects, conformal intervals, portfolio allocations, a -YAML/Parquet feature contract, tables, figures, and status reports. The anonymous -submission describes the bundle without -revealing author identity. Repository and remote-storage URLs will be disclosed -according to the journal's double-anonymous and data/code-disclosure policy. - -The body-supplement split is fixed before submission. The body keeps the CRPTO -pipeline, the alpha-to-portfolio link, the finite-grid frontier, and the -core metrics, plus the compact regret-auditability frontier. The online supplement carries -A3--A39, the conformal finalist ablation, funded-set loan audit, tail-risk -diagnostics, satisficing margins, dependence diagnostics, the CVaR/OCE -tail-constrained re-optimization (A22), the multi-distribution (A23) and online -ACI-stability (A24) diagnostics, the multi-dataset external economic replication -tables (A25--A34), the selected-policy frontier and funded-set audits -(A35--A39), MRM/fairness material, and reproduction commands. This keeps the -IJDS body focused while preserving the audit trail that reviewers need. - -\subsection{Multi-Dataset External Replication Protocol} - -The external-replication layer is narrower than a new benchmark -campaign. We reuse the frozen CRPTO recipe---CatBoost PD, calibration, train-only -WOE/IV feature screening, Mondrian conformal intervals, and the same bound-aware -robust LP---on two credit datasets with economic fields. Prosper contributes -final-status marketplace loans with observed outcome, loan amount, yield/rate, -and a temporal OOT window. Freddie/Mendeley contributes processed single-family -mortgage panels derived from the Freddie Mac loan-level ecosystem, including -12--60 month default windows and train/OOS/OOT splits. Home Credit was audited -but not promoted because it lacks a clean investment-return and exposure contract -comparable to Lending Club, Prosper, and Freddie. This gate does not re-promote -the Lending Club champion and does not claim a new exact funded-set theorem for -every external portfolio. - -% ===================================================================== -\section{Results}\label{sec:results} +\section{Data and Evaluation Design}\label{sec:data} -Table~\ref{tab:core} summarizes the paper-facing metrics. The calibrated PD -layer is not sold as a leaderboard model: AUC $0.7139$ is sufficient only -because the downstream decision consumes calibrated probabilities, not rankings -alone. Its Brier score $0.1544$ and ECE near $0.0070$ are therefore as important -as discrimination. The conformal layer over-covers marginally at the reported -levels (90\% coverage $0.9297$, 95\% coverage $0.9664$ for the conformal winner). -The portfolio layer then turns this uncertainty into an exact finite-grid -return-bound frontier. The selected policy passes the -$V\le\sqrt{\alpha}$ certificate at the tightest reported level while keeping zero -deterministic violation. +The data are Lending Club retail loans originated from 2007 through 2020. The +feature contract contains only origination-time information. Model development +and evaluation use temporal rather than random splits. \begin{table}[t] - \centering - \caption{Frozen paper-facing core metrics by layer.} - \label{tab:core} - \begin{tabular}{llr} - \toprule - Layer & Metric & Value \\ - \midrule - PD & AUC & $0.7139$ \\ - PD & Brier score & $0.1544$ \\ - PD & ECE & $0.0070$ \\ - Conformal & Coverage 90\% & $0.9297$ \\ - Conformal & Coverage 95\% & $0.9664$ \\ - Conformal & Minimum group coverage 90\% & $0.9190$ \\ - Portfolio & Body-point robust return & \$184{,}832.48 \\ - Portfolio & $V(\alpha=0.01)$ & $0.035350$ \\ - Portfolio & $\Gamma_{\mathrm{CP}}(\alpha=0.01)$ & $0.162616$ \\ - Portfolio & Markov cap at $\alpha=0.01$ & $0.345084$ \\ - Portfolio & Exact alpha violation & $0.0$ \\ - Portfolio & Declared alpha-grid pass & $8/8$ \\ - \bottomrule - \end{tabular} +\centering +\small +\caption{Temporal Lending Club design.} +\label{tab:splits} +\begin{tabular}{lrrp{0.42\textwidth}} +\toprule +Split & Period & Loans & Role \\ +\midrule +Train & Jun 2007--Mar 2017 & 1{,}346{,}311 & Fit PD model and calibrator. \\ +Conformal fit & Mar--Oct 2017 & 142{,}550 & Estimate frozen conformal recipe. \\ +Policy selection & Nov 2017 & 14{,}943 & Rank nine outcome-free policies. \\ +Calibration audit & Dec 2017 & 20{,}695 & Replay selector, then open outcomes. \\ +OOT evaluation & Jan 2018--Sep 2020 & 276{,}869 & Freeze-then-evaluate portfolio decisions. \\ +\bottomrule +\end{tabular} \end{table} -The exact certificate is an accounting claim. Here ``exact'' means the quantities -are computed directly on the frozen OOT funded set rather than approximated by a -surrogate table or visual proxy, and the deterministic part requires no distributional -assumption. The certificate's pass is the Markov safety check $V(\alpha)\le\sqrt{\alpha}$ -together with zero deterministic violation ($\sum_i w_i Y_i\le B_u(\alpha)$); it is -\emph{not} a claim of nominal $\alpha$-coverage, which the funded set does not attain -($V=0.035350>\alpha=0.01$). - -This makes $\Gamma_{\mathrm{CP}}$ more than a diagnostic line item. It is the -amount of conformal robustness the optimizer accepts in order to keep the funded -set inside the $\sqrt{\alpha}$-safe region. A credit reviewer can therefore read -$\Gamma_{\mathrm{CP}}=0.162616$ as the conformal premium carried by the selected -body point, $V=0.035350$ as the realized weighted noncoverage audit, and the -Markov cap $0.345084$ as the conservative theorem-facing risk cap. +The frozen recipe uses the most recent 75\% of the original calibration pool +(178{,}188 rows). Within it, 142{,}550 rows estimate conformal quantiles, +14{,}943 November rows select the policy, and 20{,}695 December rows form a +pre-OOT audit. The loader stores holdout outcomes separately from the 12-column +policy frame, which contains identifiers, origination context, PD endpoints, +amounts, and rates. Ranking cannot access outcomes, realized returns, +miscoverage, or the assumption-conditional Markov quantity. December first +reruns that outcome-free selector and only then joins outcomes for the decision +audit. + +The final OOT panel covers several regimes, including the 2020 disruption. We +report the full panel and five temporal slices. Each slice solves the same fixed +policy on a fresh \$1M budget; slice returns are comparable stress evaluations, +not components that sum to the full-panel value. -\begin{table}[t] - \centering - \caption{Exact certificate for the promoted funded set. A pass denotes - $V\le\sqrt{\alpha}$ with zero deterministic violation, not nominal $\alpha$-coverage.} - \label{tab:exact-certificate} - \begin{tabular}{rrrrrr} - \toprule - $\alpha$ & $\Gamma_{\mathrm{CP}}$ & $V(\alpha)$ & $\sqrt{\alpha}$ & Markov cap & Violation \\ - \midrule - $0.01$ & $0.162616$ & $0.035350$ & $0.10000$ & $0.345084$ & $0.00000$ \\ - \bottomrule - \end{tabular} -\end{table} +\section{Method}\label{sec:method} -The central empirical object is now a return-bound frontier rather than a single -winner. The consolidated frontier surface deduplicates 51{,}678 raw rows into -50{,}010 semantic policies; 27{,}508 policies both pass every level in the -declared alpha grid and exceed the declared return floor. The terminal endpoint -search alone evaluates 37{,}068 policies and 296{,}544 exact alpha checks, with -37{,}068/37{,}068 policies passing all eight alpha levels. - -The provenance of the frontier deserves one explicit sentence, because two -certified numbers coexist in the project's governance record. The declared -return floor \$170{,}464.54 is itself the realized return of the previously -certified bound-aware allocation on the same frozen chain, retained as the -frozen upstream baseline; the declared frontier is a deterministic re-evaluation -of a pre-declared finite policy grid over the \emph{same} frozen PD model -and Mondrian conformal intervals, so no upstream object (model, calibrator, -or interval) was regenerated when the body point moved from the floor to -\$184.8K. The earlier certificate is not discarded: it becomes the floor -that every eligible frontier policy must beat, which is why the frontier is -reported with the floor surplus rather than as a replacement champion. +\subsection{Calibrated PD and Exact Conformal Replay} + +Let $p_i\in[0,1]$ be calibrated PD. The frozen CatBoost layer has AUC 0.7139, +Brier score 0.1544, and expected calibration error about 0.0070. These values +are not a leaderboard claim; probability quality matters because the economic +objective consumes PD. + +The conformal recipe partitions calibrated scores into five score-quantile +Mondrian cells. Calibration loan $j$ receives scaled residual +\begin{equation}\label{eq:score} +s_j=\frac{|Y_j-p_j|}{\sqrt{\max\{p_j(1-p_j),10^{-6}\}}}. +\end{equation} +Each cell uses the finite-sample higher quantile at frozen used alpha 0.095 for +target $\alpha=0.10$. Recorded group and temporal factors may widen but never +narrow an interval. The resulting upper endpoint is clipped to $[0,1]$. + +The replay reconstructs every setting from the frozen result payload. At the +90\% reference level it reproduces stored point, lower, and upper vectors with +maximum absolute error below $6.67\times10^{-16}$. This matters because an +earlier exploratory sweep scaled 90\% radii with average widths from another +conformal family; those values are not used in this manuscript. + +\subsection{One Portfolio Policy} + +Let $a_i$ be loan amount, $x_i\in[0,1]$ the funded fraction, $r_i$ the coupon, +$L=0.45$ loss given default, $B=1{,}000{,}000$, and $\tau$ risk tolerance. +CRPTO solves +\begin{equation}\label{eq:portfolio} +\begin{aligned} +\max_x\quad & \sum_i x_i a_i(r_i-p_iL) \\ +\text{s.t.}\quad +& \sum_i x_i a_i\le B, \\ +& \sum_i x_i a_iq_i\le\tau\sum_i x_i a_i, \\ +& 0\le x_i\le\bar x_i, +\end{aligned} +\end{equation} +with the existing eligibility and concentration constraints. The selected +policy uses $\tau=0.17$ and Equation~\eqref{eq:midpoint}. The objective uses +$p_i$, not $q_i$: expected economics and uncertainty feasibility are separate +contracts. The matched point-PD baseline changes only the risk score to +$q_i=p_i$ while holding candidates, budget, concentration, LGD, solver, and +risk tolerance fixed. + +For funded-exposure weights $w_i=x_ia_i/\sum_jx_ja_j$, define +\begin{equation}\label{eq:gamma} +\begin{aligned} +\Gamma_{\mathrm{CP}}&=\sum_iw_i(u_i-p_i),\\ +\Gamma_{\mathrm{int}}&=\sum_iw_i(q_i-p_i),\\ +\Gamma_{\mathrm{res}}&=\sum_iw_i(u_i-q_i),\\ +B_u&=\sum_iw_iu_i. +\end{aligned} +\end{equation} +For the midpoint policy, +$\Gamma_{\mathrm{int}}=\Gamma_{\mathrm{res}}=\Gamma_{\mathrm{CP}}/2$. + +\subsection{Calibration-Only Final Selector} + +The candidate set crosses $\tau\in\{0.15,0.17,0.19\}$ with +$\gamma\in\{0.25,0.50,0.75\}$ in +$q_i=p_i+\gamma(u_i-p_i)$. A candidate is eligible when the solver is optimal, +at least 99.9\% of budget is allocated, the effective-PD cap holds, and +\begin{equation}\label{eq:screen} +B_u\le0.28 +\end{equation} +on November 2017. Among eligible candidates, the rule maximizes expected +point-PD objective. Five of nine pass; the selected row is +$\tau=0.17,\gamma=0.50$. The endpoint cap is deterministic and does not use +the weighted-validity assumption below. The midpoint remains selected for +every cap in $[0.259036,0.290491)$, and an outcome-free December replay selects +the same row. \begin{table}[t] - \centering - \caption{Pool93 finite-grid return-bound frontier. \texttt{terminal} rows come from - the conservative endpoint sweep; \texttt{micro-ext} rows come from the body-cap local - extension around the selected point. All rows are finite-grid points, not continuous - optima.} - \label{tab:pool93-frontier} - \resizebox{\textwidth}{!}{% - \begin{tabular}{llrrrrr} - \toprule - Policy role & Source & Realized return & $V(0.01)$ & - $\Gamma_{\mathrm{CP}}$ & Markov cap & Pass \\ - \midrule - Minimum Markov-cap endpoint & terminal & \$170{,}467.27 & $0.031875$ & $0.095719$ & $0.273036$ & $8/8$ \\ - Low-cap balanced endpoint & terminal & \$171{,}006.20 & $0.031875$ & $0.097190$ & $0.274789$ & $8/8$ \\ - Highest return under cap $\le 0.30$ & terminal & \$173{,}314.04 & $0.035875$ & $0.115400$ & $0.294580$ & $8/8$ \\ - Highest return under cap $\le 0.345$ & micro-ext & \$184{,}800.41 & $0.035350$ & $0.162562$ & $0.344996$ & $8/8$ \\ - Body/default balanced point & micro-ext & \$184{,}832.48 & $0.035350$ & $0.162616$ & $0.345084$ & $8/8$ \\ - Highest return under cap $\le 0.36$ & micro-ext & \$186{,}050.73 & $0.037750$ & $0.174600$ & $0.358685$ & $8/8$ \\ - Max-return economic endpoint & micro-ext & \$223{,}458.14 & $0.069575$ & $0.457438$ & $0.510753$ & $8/8$ \\ - \bottomrule - \end{tabular} - }% +\centering +\small +\caption{Temporally separated selector examples; the supplement reports all nine rows.} +\label{tab:selector} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lrrrrrl} +\toprule +Candidate & $\tau$ & $\gamma$ & Nov. objective & Nov. $B_u$ & Dec. $B_u$ & Status \\ +\midrule +Low guardrail & 0.17 & 0.25 & \$109{,}885.56 & 0.375105 & 0.396751 & Ineligible \\ +Selected midpoint & 0.17 & 0.50 & \$99{,}387.12 & 0.259036 & 0.262082 & Selected twice \\ +Conservative blend & 0.17 & 0.75 & \$93{,}760.13 & 0.202259 & 0.203504 & Eligible \\ +Higher tolerance & 0.19 & 0.50 & \$102{,}671.87 & 0.290491 & 0.294861 & Ineligible \\ +\bottomrule +\end{tabular}} \end{table} -Table~\ref{tab:pool93-frontier} gives the manuscript its decision geometry. The -body/default point is not the highest-return point and not the tightest-bound -point; it is the balanced point selected by the declared return-bound lens. The -endpoint at Markov cap $0.273036$ shows how conservative the certified frontier -can become while preserving the return floor, and the \$223.5K endpoint shows -the economic return available when the committee accepts a looser cap. The -supplement reports the full frontier table and traceability details. - -The funded-set under-coverage remains structural rather than a calibration-draw -effect. With $n_{\mathrm{cal}}=237{,}584$ calibration loans, the split-conformal -conditional-coverage result makes marginal coverage highly stable around the -nominal level \citep{vovk2005,angelopoulos2023}. The residual funded-set $V$ is a -test-side, portfolio-selection quantity. That is why the paper reads the safety -level at $\sqrt{\alpha}$ under Assumption~\ref{asm:weighted-validity}, and why -the frontier reports $V$, $\Gamma_{\mathrm{CP}}$, endpoint budget, Markov cap, and -return together instead of promoting a standalone coverage number. +The selector's input whitelist contains no outcomes or +assumption-conditional quantities. Earlier research iterations used the OOT +corpus; the narrow claim is that this final ranking code path does not. + +\subsection{Accounting and Statistical Boundary} + +Let $Z_i=\mathbf1\{Y_i>u_i\}$ and $V=\sum_iw_iZ_i$. Since +$Y_i\le u_i+Z_i$ for every funded loan, +\begin{equation}\label{eq:accounting} +\sum_iw_iY_i\le B_u+V +\end{equation} +holds deterministically. It is funded-set accounting, not a coverage theorem. + +If one additionally assumes weighted funded-set validity, +$\mathbb E[V]\le\alpha$, Markov's inequality gives +\begin{equation}\label{eq:markov} +\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt{\alpha}\right) +\le\sqrt{\alpha}. +\end{equation} +The assumption does not follow from marginal split conformal because the +optimizer chooses $w_i$ from $p_i$ and $u_i$. Equation~\eqref{eq:markov} is +therefore secondary, assumption-conditional sensitivity. The supplement gives +the proof and selected-set boundary \citep{hegazy2025valid_selection_conformal_sets}. -\begin{table}[t] - \centering - \caption{Reviewer claim checks in the main manuscript.} - \label{tab:reviewer-checks} - \resizebox{\textwidth}{!}{% - \begin{tabular}{lll} - \toprule - Reviewer concern & Body answer & Primary evidence \\ - \midrule - ``This is only a classifier.'' & The claim is decision auditability, not AUC leadership. & Exact funded-set certificate and A35 frontier. \\ - ``CP + RO already exists.'' & CRPTO instantiates the idea for frozen credit PD models, funded-set governance, and Lending Club payoffs. & Closest-work boundary and bound claim stack. \\ - ``Adaptive selection breaks coverage.'' & The theorem states weighted funded-set validity as an assumption and then audits the frozen selection exactly. & Assumption map, validity ladder, and A23 diagnostics. \\ - ``The selected policy is cherry-picked.'' & The selected point comes from a consolidated finite frontier with 50{,}010 deduplicated semantic policies and 27{,}508 eligible all-alpha above-floor policies. & Frontier table and governance files. \\ - \bottomrule - \end{tabular} - }% -\end{table} +\section{Results}\label{sec:results} -\FloatBarrier -\subsection{Multi-Dataset External Economic Replication} +\subsection{Exact 90\% Conformal Evidence} -The natural generalization question after the Lending Club audit is whether the -recipe still works outside the champion panel. Table~\ref{tab:external-replication} -answers that question without changing the champion: the same frozen recipe is -applied to two external credit products. -Both pass the conformal gates and both return positive robust LP objectives. +At target $\alpha=0.10$, exact OOT coverage is 0.934836, average width is +0.788879, minimum score-partition coverage is 0.926310, minimum letter-grade +coverage is 0.926797, and 51.7873\% of upper endpoints equal one. The intervals +are conservative and broad, as expected for a binary outcome on the probability +scale. -\begin{table}[t] - \centering - \caption{External economic replications using the frozen CRPTO recipe.} - \label{tab:external-replication} - \resizebox{\textwidth}{!}{% - \begin{tabular}{llrrrrrrr} - \toprule - Dataset & Product & Rows & Default & AUC & Cov. 90\% & Cov. 99\% & OOT cand. & Robust LP \\ - \midrule - Prosper & Marketplace loans & 54{,}807 & 30.92\% & 0.7074 & 0.9205 & 0.9943 & 10{,}531 & \$199{,}419 \\ - Freddie FM48 & Mortgages & 3{,}173{,}355 & 1.45\% & 0.7839 & 0.9745 & 0.9907 & 1{,}396{,}053 & \$1{,}291{,}228 \\ - \bottomrule - \end{tabular} - }% -\end{table} +The exact alpha sensitivity explains why the paper does not headline a 99\% +interval. At target alpha 0.01, coverage is 0.996720, but average width is +0.988215 and 93.5424\% of upper endpoints equal one. Such endpoints carry +almost no ranking information for a portfolio. The selected 90\% level is the +frozen recipe's reference level and preserves more decision resolution. -Prosper uses its full 10{,}531-loan OOT economic universe. Freddie is evaluated -on 1{,}396{,}053 OOT economic candidates; a sparse all-candidate LP returns the -same robust objective as the large top screens, with worst funded rank 551 and -zero funded loans outside the top-250{,}000 screen. This is an exhaustiveness -audit on the external LP solve, not a new exact funded-set certificate for -Prosper or Freddie. - -\begin{figure}[ht] - \centering - \includegraphics[width=0.94\textwidth]{crpto_fig22_external_replication.pdf} - \caption{External CRPTO replications preserve predeclared global conformal gates and positive - robust LP value on two materially different credit products.} - \label{fig:external-replication} -\end{figure} +\subsection{Pre-OOT Selector and Decision Audit} -\FloatBarrier -The external layer also surfaces a result a single-dataset champion cannot show. -The signed price of robustness---using the same convention as the Lending Club -field, $(\mathrm{nonrobust}-\mathrm{robust})/\mathrm{nonrobust}$---is a -\emph{positive} premium under frozen application, and across the four frozen -external applications it is ordered by panel default rate (Table~\ref{tab:price-of-robustness}, -Figure~\ref{fig:price-scaling}). Within Freddie, the high-default red segment pays more -than green; across datasets, Prosper's $30.92\%$ default panel pays the largest premium. -This is a pattern across two datasets---three Freddie default-window segments plus -Prosper---not a scaling law: it is consistent with the mechanism (higher default risk -widens the conformal intervals, so the robust worst case discounts more return, and -discrimination (AUC) does not order the premium on its own), but four points cannot -establish a general law. On the \emph{selected} Lending Club champion the -signed price is favorable ($-10.56\%$), because the bound-aware search found a -robust funded set that also wins expected return. The measured summary is more -modest: in these applications, the conformal robust layer is economically -bounded. Under blind application it costs a single-digit to low-double-digit -premium; under selection it can be favorable. +November selects the midpoint policy from five eligible rows. Applying the +same outcome-free rule to December again selects \texttt{linear-005}; outcomes +are opened only after that replay. \begin{table}[t] - \centering - \caption{Price of robustness by application, ordered by panel default - rate. The selected Lending Club champion is $-10.56\%$ (favorable under - selection, not blind application).} - \label{tab:price-of-robustness} - \begin{tabular}{lrr} - \toprule - Application & Panel default & Price of robustness \\ - \midrule - Freddie FM48 (green) & 0.58\% & $+1.00\%$ \\ - Freddie FM48 (combined) & 1.45\% & $+1.09\%$ \\ - Freddie FM48 (red) & 2.97\% & $+2.37\%$ \\ - Prosper final-status & 30.92\% & $+9.46\%$ \\ - \bottomrule - \end{tabular} +\centering +\small +\caption{Independent December 2017 post-selection decision audit.} +\label{tab:calibration-audit} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lrrrrr} +\toprule +Policy & Funded & Realized return & Weighted default & Miscoverage & $B_u$ \\ +\midrule +Selected 50/50 CRPTO & 193 & \$53{,}313.05 & 0.145650 & 0.124925 & 0.262082 \\ +Conservative 75\% blend & 191 & \$38{,}379.50 & 0.155250 & 0.134525 & 0.203504 \\ +Point-PD matched $\tau$ & 169 & \$89{,}732.35 & 0.185650 & 0.058300 & 0.888071 \\ +\bottomrule +\end{tabular}} \end{table} -\begin{figure}[t] - \centering - \includegraphics[width=0.78\textwidth]{crpto_fig25_price_of_robustness_scaling.pdf} - \caption{Across the four frozen external applications, the price of robustness - is ordered by panel default risk; the selected Lending Club champion sits - below zero as a favorable reference.} - \label{fig:price-scaling} -\end{figure} +The selected policy remains below its operational tolerance +($0.145650<0.17$) but misses nominal funded-set coverage +($V=0.124925>0.10$). Its deterministic accounting right-hand side is +$B_u+V=0.387007$. Thus outcome-free, month-stable selection does not create +selected-set conformal validity. This negative audit motivates the two-layer +reporting of deterministic endpoint exposure and observed miscoverage. + +\subsection{Full OOT Funded-Set Audit} -% ===================================================================== -\FloatBarrier -\section{Robustness and Comparators}\label{sec:robustness} - -The first robustness concern is temporal leakage. CRPTO addresses it through -out-of-time splits, temporal backtesting, and a strict distinction between -calibration, test, and paper-facing outputs. The paper does not claim that a Lending -Club static panel substitutes for future originations after the retail platform -closed. It claims that within the available historical panel, the promoted policy -survives the documented temporal validation design. The strict temporal holdout -in supplement A9 reports both temporal slices passing the exact check, strengthening -that validation claim without reopening the champion. - -The second concern is whether conformal uncertainty is doing decision work or only -adding conservative decoration. The answer is visible in the portfolio frontier. -Policies are evaluated by return, exact alpha pass/fail, weighted miscoverage, and -$\Gamma_{\mathrm{CP}}$; the promoted point is selected because it earns the highest -robust return inside the exact feasible region. This differs from a workflow where -conformal intervals are plotted after the optimizer has already chosen a point-PD -allocation. - -The supplement also carries the reviewer-facing robustness checks: nested holdout, -strict temporal holdout, exact evaluation of conformal finalists, uncertainty-set -baselines, and the selected-policy finite-grid frontier. Diagnostics that depend on -the selected funded-loan composition are split explicitly: selected-allocation checks -stay diagnostic, while legacy tail-frontier tables remain -diagnostic machinery rather than body selectors. - -The third concern is whether a decision-focused or SPO+ model would be a stronger -baseline. CRPTO treats SPO+ as an important comparator, but not as the same governance -object. Decision-focused training can reduce regret relative to an optimization loss, -while CRPTO prioritizes calibrated uncertainty, traceable risk controls, and exact -funded-set checks. The manuscript therefore does not claim to dominate SPO+ on every -regret metric; it claims a different auditability/economic trade-off. - -\subsection{Regret-Auditability Frontier}\label{sec:regret} - -The SPO+ comparator makes the trade-off sharp. In the committed A19/Figure~\ref{fig:regret} -results, SPO+ reduces mean regret from $0.425896$ to $0.216837$, a $49.09\%$ -improvement over the two-stage baseline (Wilcoxon $p=1.39\times10^{-164}$). A later -PyEPO 1.3.7 paired rerun independently reports the same conclusion under a slightly -different protocol ($0.358073$ to $0.184366$, $48.51\%$; Wilcoxon -$p=3.80\times10^{-163}$), so we treat it as a curated closeout note rather than as the -numeric source for A19. The CRPTO robust point has higher decision regret ($0.947429$) -because it is not trained to minimize regret; it is constructed to expose and control -predictive uncertainty before funding. The frontier is therefore not a single -leaderboard. It asks what the method buys besides regret: temporal coverage above -target, an exact funded-set $\alpha=0.01$ pass, and a finite-grid return-bound -frontier. +The fixed midpoint policy allocates the full \$1M budget across 308 loans. Its +expected point-PD objective is \$168{,}271.56; realized return is +\$179{,}327.59. Weighted default is 0.039375 and weighted miscoverage is +0.036875. \begin{table}[t] - \centering - \caption{Regret-auditability frontier: CRPTO trades synthetic benchmark regret for - verifiable risk controls. Regret is the A19/PyEPO decision-loss scale, not realized - dollar return on the \$1M funded portfolio.} - \label{tab:regret} - \resizebox{\textwidth}{!}{% - \begin{tabular}{lrrrr} - \toprule - Method & Mean regret & Regret $\Delta$ vs. two-stage & Realized funding value & Verifiable risk controls \\ - \midrule - Two-stage baseline & $0.425896$ & $0.0\%$ & not certified & $0/3$ \\ - SPO+ & $0.216837$ & $49.09\%$ lower & not certified & $0/3$ \\ - CRPTO robust & $0.947429$ & $122.46\%$ higher & \$184.8K & $3/3$ \\ - \bottomrule - \end{tabular} - }% +\centering +\small +\caption{Exact full-OOT audit of the selected midpoint policy.} +\label{tab:active} +\begin{tabular}{lr} +\toprule +Quantity & Value \\ +\midrule +Weighted point PD & 0.081949 \\ +Weighted midpoint score & 0.170000 \\ +$\Gamma_{\mathrm{CP}}$ & 0.176102 \\ +$\Gamma_{\mathrm{int}}$ & 0.088051 \\ +$\Gamma_{\mathrm{res}}$ & 0.088051 \\ +Endpoint budget $B_u$ & 0.258051 \\ +Observed accounting bound $B_u+V$ & 0.294926 \\ +Conditional Markov threshold & 0.574279 \\ +\bottomrule +\end{tabular} \end{table} -The regret column must be read with its protocol in mind, and the last two -columns prevent a one-dimensional reading. Mean regret comes from a separate -decision-regret experiment (A19/PyEPO) run on small synthetic optimization -instances (50 items, budget 15, five seeds), scoring each method on a normalized -decision-loss scale rather than on the \$1M funded portfolio; on that scale SPO+ -is the low-regret method by construction. The favorable price of robustness above -and the higher CRPTO regret here are not in tension---they are two different -measurements (a real \$1M funded set versus a synthetic regret benchmark). The -right-hand columns report what the credit decision actually delivers: only CRPTO -produces a budgeted funded set with a certified realized return (\$184.8K on -the \$1M budget) and the three verifiable risk controls. The regret comparison is -therefore about the synthetic benchmark task, not the quality of the funded loans -in the \$1M credit portfolio. +The observed weighted outcome 0.039375 is below the exact accounting right-hand +side 0.294926. Under the additional weighted-validity assumption, the 0.574279 +event threshold has probability bound $\sqrt{0.10}=0.316228$. This loose +statement is not a direct default cap; the operational controls are $\tau$, +$q_i$, and the funded-set diagnostics. -\begin{figure}[t] - \centering - \includegraphics[width=0.58\textwidth]{crpto_fig15_regret_auditability_frontier.pdf} - \caption{CRPTO sits on the auditable-risk-control corner, while SPO+ occupies the - low-regret corner.} - \label{fig:regret} -\end{figure} +A fixed-allocation bootstrap over 31 origination-month clusters gives a 95\% +return interval of \$163{,}421.14--\$193{,}551.65 from 5{,}000 draws. A +funded-loan sensitivity gives \$162{,}706.17--\$193{,}924.74. Neither resamples +the model, conformal recipe, selector, or optimizer. The month-level scheme is +primary because the temporal results reject a naive independence story. -CRPTO is also close in spirit to a recent line that gives distribution-free, -finite-sample guarantees jointly on miscoverage and decision regret, tracing a -miscoverage--regret Pareto frontier for robust predict-then-optimize policies -\citep{zhou2025credo,zhou2026creme}. That work is the general theory of the -frontier in Figure~\ref{fig:regret}: it calibrates a robustness level against a -cost--risk preference for an abstract optimization family. CRPTO is the -complementary, applied object: it does not propose a new frontier estimator, but -instantiates one corner of that frontier on a frozen, production-style -credit-risk model, with a named conformal robustness premium -$\Gamma_{\mathrm{CP}}$, an exact funded-set certificate, finite-grid portfolio -frontier, and model-risk lineage that a credit committee can inspect. - -\FloatBarrier -\subsection{Tail Risk and Distribution Robustness}\label{sec:tail-dist} - -Two reviewer questions deserve a body-level answer: what does the selected policy -give up on the tail, and does its coverage hold once the evaluation is sliced by -grade? The return-bound frontier answers the first question directly: lower -Markov caps are available at lower return, while the selected policy sits on the -declared frontier. The supplement then checks the funded-grade mix, -selected-allocation tail repricing, dependence sensitivity, and fixed-allocation -bootstrap interval. These rows explain the selected policy's risk profile; they -do not add a hidden CVaR/OCE or bootstrap selector. +\subsection{Matched Comparators} \begin{table}[t] - \centering - \caption{Tail-risk and distribution-robustness checks for the selected decision.} - \label{tab:tail-scope} - {% - \small - \setlength{\tabcolsep}{4pt} - \renewcommand{\arraystretch}{1.18} - \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.26\textwidth}>{\raggedright\arraybackslash}p{0.36\textwidth}>{\raggedright\arraybackslash}p{0.30\textwidth}@{}} - \toprule - Reviewer question & Body answer & Boundary \\ - \midrule - Is the point only high-return? & The frontier shows safer lower-return choices and higher-return looser-cap choices. & Finite grid, not a continuous optimum. \\ - What loans does it fund? & The supplement reports funded exposure by grade. & Business mix, not protected-class fairness certification. \\ - What happens in the tail? & The selected allocation is repriced under LGD and tail summaries. & Risk profile only; tail risk is not the selector. \\ - Does dependence change the bound? & Cluster sensitivity recomputes tighter assumptions. & Sensitivity only; Markov remains the body theorem. \\ - Is the return estimate fragile? & A fixed-allocation bootstrap reports contribution intervals. & Does not resample the model, solver, or search. \\ - Does coverage survive slices? & Grade, distribution, and online-style checks stress the frozen intervals. & Diagnostics, not universal conditional or live coverage. \\ - \bottomrule - \end{tabular} - }% +\centering +\small +\caption{Full-OOT matched decision comparison.} +\label{tab:matched} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lrrrrrr} +\toprule +Policy & Funded & Return & Default & Miscoverage & $B_u$ & Threshold \\ +\midrule +Selected 50/50 CRPTO & 308 & \$179{,}327.59 & 0.039375 & 0.036875 & 0.258051 & 0.574279 \\ +Conservative 75\% blend & 312 & \$172{,}939.50 & 0.035875 & 0.035875 & 0.200396 & 0.516624 \\ +Point-PD matched $\tau$ & 225 & \$196{,}369.14 & 0.118400 & 0.041900 & 0.921317 & 1.237545 \\ +\bottomrule +\end{tabular}} \end{table} -On distribution robustness, the worst per-grade 90\% coverage on the frozen -intervals is grade E at $0.9004$, still above the $0.90$ target. Supplement A23 -reports marginal coverage $0.9293$ on its multi-distribution evaluation slice, -while the promoted interval summary in Table~\ref{tab:core} reports $0.9297$; -these are distinct cuts through the same intervals. No grade falls -below target, so the conservative marginal coverage is not hiding a failing segment; the thinnest -grade$\times$vintage cells are where a future group-weighted or multi-distribution -recalibration would matter, marked as future work rather than a present guarantee. - -\subsection{Managerial Implication}\label{sec:managerial} - -For a credit-risk committee, CRPTO turns a model into a decision -conversation. The committee can pick a risk cap $\tau$, inspect how a policy -parameter $\gamma$ changes the funded set, read $\Gamma_{\mathrm{CP}}$ as the -conformal robustness premium paid by that funded set, and compare $V(\alpha)$ -with the stated bound tolerance. The method therefore supports a practical -question: whether the realized economic return justifies the signed price of -robustness, which on this evaluation is favorable rather than a cost. If the -committee wants lower regret, the SPO+ corner is visible; if it wants stronger -validity language, the validity ladder states the new -calibration protocol that would be required. That separation is the managerial -value of the paper. - -The online supplement contains the robustness package: nested holdout, segment-period -sensitivity, decision-aware conformal selector checks, synthetic-shift diagnostics, -conformal finalist exact evaluation, tail-risk OCE/CVaR diagnostics, satisficing -margins, dependency clusters, bootstrap funded-set metrics, budget/LGD/cap sensitivity, -and finite-grid frontier summaries by policy family, plus the A19 -regret-auditability frontier, A20 tail-risk diagnostic audit, A21 cluster-bound -tightening audit, A22 CVaR/OCE tail-constrained re-optimization, and A23--A24 -multi-distribution and online (ACI) conformal-stability diagnostics, plus -A25--A34 external economic replication, exhaustiveness, interval, subperiod, -and sensitivity audits on Prosper and Freddie/Mendeley, and A35--A39 selected-policy -frontier, composition, tail-risk, concentration, and bootstrap audits. - -% ===================================================================== -\section{Discussion}\label{sec:discussion} - -CRPTO is useful precisely because it stays close to the operational reality of -credit-risk analytics. Many institutions already have calibrated PD models and -portfolio policies. Replacing them with an end-to-end decision-focused learner may be -scientifically attractive but organizationally difficult. CRPTO offers a middle path: -keep the predictive model auditable, quantify uncertainty with a finite-sample -conformal layer, and make the optimizer pay attention to the upper end of plausible -default risk. - -The external replications also change how to read the price of robustness. Across -the frozen external applications, the premium is ordered by panel default risk -rather than by discrimination. That pattern reframes robustness as a -panel-specific premium to be measured, not a fixed toll to be assumed. It also -tempers the contribution: the favorable Lending Club value reflects champion -selection, so the transferable claim is bounded measurement under blind -application, not a universal free lunch. - -The limits are equally important. CRPTO does not prove that any one public dataset -is a universal proxy for modern credit origination. The Prosper and -Freddie/Mendeley replications reduce the single-dataset concern, but they are still -static historical panels rather than live deployment evidence. The external panels also make the conditional-coverage -caveat concrete: on Freddie the all-group minimum coverage is driven by tiny sparse -Mondrian cells, and the high-default red segment misses the strict $\alpha = 0.01$ -gate at $0.9850$; both are reported as sensitivity evidence rather than promoted -as conditional guarantees. CRPTO does not claim legal fairness certification, -because the public data lack direct protected attributes. It does not assert that -robust conformal policies dominate all decision-focused learners on regret. - -The next research directions are clear and are now separated into paper -improvements versus new CRPTO protocols. The current paper includes the safe -journal-strengthening package: OCE/CVaR as a tail-risk audit, satisficing as -margin evidence, SPO+ as the low-regret corner of the regret-auditability -frontier, and dependence as a formal caveat. The larger scientific upgrades are -useful in the present manuscript only when they sharpen this boundary. -Table~\ref{tab:upgrade-map} states that boundary directly. +Relative to point PD, selected CRPTO gives up \$17{,}041.55, or 8.678\% of +realized return. Weighted default falls by 7.9025 percentage points, +miscoverage by 0.5025 percentage points, and the endpoint-plus-Markov threshold +by 66.3266 percentage points. The default contrast is much larger than the +miscoverage contrast: the guardrail changes the funded loans but does not create +a dramatic selected-set coverage gain. + +The 75\% blend lowers default by 0.35 percentage points and the threshold by +0.057655 relative to the midpoint, but costs another \$6{,}388.08. The +calibration selector chooses the midpoint because it has the highest expected +objective under the declared screen, not because it dominates every risk +metric. + +\subsection{Temporal Heterogeneity} \begin{table}[t] - \centering - \caption{Scientific upgrade map for the current CRPTO paper.} - \label{tab:upgrade-map} - {% - \small - \setlength{\tabcolsep}{4pt} - \renewcommand{\arraystretch}{1.12} - \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.22\textwidth}>{\raggedright\arraybackslash}p{0.36\textwidth}>{\raggedright\arraybackslash}p{0.34\textwidth}@{}} - \toprule - Upgrade path & What the current paper can claim & What would require a new result \\ - \midrule - Tail-aware selection & A20--A22 and A37 show the tail trade-off and selected-allocation tail profile. & A predeclared CVaR/OCE selector with its own exact funded-set audit. \\ - Prospective selection & Nested holdout and the finite declared grid reduce post-selection ambiguity. & A fully prospective search/evaluation split after all policy rules are frozen. \\ - Multi-distribution or online validity & A23--A24 diagnose grade, distribution, and vintage stress on frozen intervals. & A new calibration protocol that targets group-weighted, multi-source, or live sequential validity. \\ - Decision-focused conformal learning & A19 shows SPO+ as the low-regret comparator and CRPTO as the auditable corner. & An end-to-end learner that jointly optimizes decision loss and emits a conformal funded-set certificate. \\ - \bottomrule - \end{tabular} - }% +\centering +\small +\caption{Fixed-policy temporal stress evaluation; each row uses a fresh \$1M budget.} +\label{tab:temporal} +\begin{tabular}{lrrrr} +\toprule +Period & CRPTO return & Point-PD return & CRPTO default & Point-PD default \\ +\midrule +2018H1 & \$92{,}530.73 & \$118{,}101.99 & 0.106703 & 0.190825 \\ +2018H2 & \$156{,}185.51 & \$95{,}603.58 & 0.026725 & 0.236728 \\ +2019H1 & \$123{,}590.69 & \$144{,}281.46 & 0.077325 & 0.170275 \\ +2019H2 & \$110{,}251.95 & \$256{,}966.20 & 0.103250 & 0.023775 \\ +2020+ & \$99{,}689.54 & \$218{,}629.14 & 0.083775 & 0.016900 \\ +\bottomrule +\end{tabular} \end{table} -Optimized OCE/CVaR objectives, online conformal methods, multi-distribution -conformal validity, utility-directed or decision-theoretic conformal variants -\citep{cortesgomez2025utility,lekeufack2023cdt}, causal variants, broader -asset-class panels, and prospective multi-period origination studies are all -valuable. They become stronger paper claims only with new protocols, data, or -proofs. In the current submission, their role is to make the frozen CRPTO -contribution easier to locate: an auditable post-hoc predict-then-optimize -certificate, not a universal decision-learning framework. +CRPTO is economically and statistically attractive in 2018H2, when the point- +PD portfolio concentrates in realized defaults. In 2019H2 and 2020+, point PD +earns more and defaults less. The result is therefore a full-period return-risk +trade-off with temporal heterogeneity, not a universal robustness premium. + +\subsection{Funded-Set Composition} + +Grades C and D represent 31.36\% and 58.68\% of funded exposure. Grade F is +only 1.30\% but has realized default 0.269231. The supplement reports every +letter grade and reconciles to the full allocation. Letter grade is recovered +from the original subgrade field; the score-quantile conformal group is stored +separately. This is a business-risk composition audit, not legal fair-lending +certification. + +\section{Robustness, Comparators, and Implications}\label{sec:implications} + +The online supplement keeps diagnostics that help challenge the one active +method without turning the paper into several methods: exact alpha saturation, +all selector cells, temporal results, grade composition, bootstrap, and matched +comparators. Earlier OCE/CVaR, dependence, online-style, SPO+, Prosper, and +Freddie/Mendeley analyses are supporting diagnostics or external context. They +do not select the active policy. + +For a credit committee, the useful control is not the Markov threshold alone. +The committee chooses a conformal level, a transparent blend $\gamma$, and a +portfolio tolerance $\tau$, then reads economic cost, endpoint exposure, +realized default, and temporal behavior together. Here the midpoint has a clear +interpretation: half of the conformal premium enters the enforceable risk score +and half remains visible as residual endpoint exposure. + +The matched baseline turns that interpretation into a decision. A committee +that accepts the full-period evidence pays 8.678\% of realized return for a +7.9025 percentage-point default reduction. A more conservative committee can +use the 75\% blend and pay another roughly \$6.4K. A committee emphasizing +2019H2 or 2020 should reject the static rule or require a new temporal +recalibration protocol. CRPTO exposes that choice rather than hiding it behind +a single score. + +\section{Reproducibility and Limitations}\label{sec:limitations} + +The research bundle versions code, manuscript sources, configurations, tables, +figures, and governance files. Heavy data and model artifacts are stored +separately and checked by hashes. The evidence builder regenerates A35--A40 +from frozen exact-alpha and policy outputs. Claim-sync tests verify that the +body, supplement, and this official TeX surface share policy settings, selector +counts, and numeric anchors. + +Several limitations bound the contribution. Lending Club retail origination +ended in 2020, so the historical panel cannot establish live performance. The +final selector is outcome-free with respect to OOT ranking, but earlier project +development inspected the same OOT corpus. The evaluation is retrospective, +not prospective. The conformal intervals are broad because the outcome is +binary; more than half of 90\% OOT upper endpoints equal one. Mondrian coverage +does not imply validity under optimizer-selected funded weights. The December +audit makes this visible: the same policy is reselected without outcomes, yet +funded-set miscoverage is 0.124925. The Markov statement therefore requires an +explicit assumption and is absent from policy selection. Temporal slices show +that point PD can dominate both return and default. Public data also do not +support a legal fair-lending certification or a causal interpretation. + +A focused next step is a genuinely prospective or formally selection-valid +protocol frozen before a new evaluation period +\citep{farinhas2024nonexchangeable_crc,hegazy2025valid_selection_conformal_sets, +zhou2026creme}. +That extension is not hidden as an acceptance criterion for this retrospective +decision audit. + +For double-anonymous review, author-identifying repository URLs are omitted. +Code/data access and artifact instructions can be disclosed under the journal's +policy without embedding credentials or identity in the anonymous PDF. -% ===================================================================== \section{Conclusion}\label{sec:conclusion} -Can finite predictive uncertainty change a funding decision in a way a reviewer -can audit? CRPTO's answer is yes, provided the statistical boundary is stated -and the decision record is frozen. On the Lending Club out-of-time panel the -selected policy earns \$184{,}832.48 on a \$1M budget while passing -the exact empirical $\alpha=0.01$ funded-set audit, and it lies on a declared -finite-grid return-bound frontier with 50{,}010 deduplicated semantic policies -and 27{,}508 all-alpha above-floor policies rather than at a single lucky point. -The external -Prosper and Freddie/Mendeley replications show that the recipe travels beyond the -Lending Club panel and expose an economically interpretable regularity: the price -of robustness is a bounded premium ordered by panel default risk across the -frozen external applications. The -contribution is scoped as an auditable post-hoc decision certificate, not a new -end-to-end learner or a live-deployment study, and every reported number is -regenerable from frozen evidence. - -% Reproducibility/companion disclosure is kept for the cover letter / non-anonymous -% version, not the double-anonymous body. +CRPTO shows how a frozen credit model can become an auditable portfolio +decision without a maze of policy variants. An exact 90\% conformal replay +produces $u_i$; the midpoint $q_i=(p_i+u_i)/2$ constrains risk; and a nine-cell +November selector fixes $\tau=0.17$ under $B_u\le0.28$. December independently +selects the same rule and then misses nominal funded-set coverage; policy +stability is therefore not relabeled as conformal validity. On the +full panel, the policy earns \$179{,}327.59, with weighted default 0.039375, +miscoverage 0.036875, $\Gamma_{\mathrm{CP}}=0.176102$, +$\Gamma_{\mathrm{res}}=0.088051$, endpoint 0.258051, observed accounting bound +0.294926, and conditional Markov threshold 0.574279. Against matched point PD, +it pays 8.678\% of realized return for a 7.9025 percentage-point default +reduction. Temporal reversals keep the claim narrow: CRPTO is an inspectable +retrospective return-risk guardrail, not a universal winner, prospective +deployment guarantee, or new credit-scoring leaderboard. \bibliographystyle{informs2014} \bibliography{../../book/references} diff --git a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md index 12fd04c..9e77443 100644 --- a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md +++ b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md @@ -1,70 +1,70 @@ # IJDS Submission Roadmap - Target 2026-08-10 -This roadmap keeps the CRPTO submission work aligned with INFORMS Journal on -Data Science rather than with a generic machine-learning or operations-research -paper. The target date is August 10, 2026. IJDS regular submissions are rolling; -the date is an internal quality gate, not an external deadline. +The date is an internal quality gate; IJDS submissions are rolling. -Official sources to recheck before freezing: +Official sources to recheck in the submission week: -- Submission guidelines: -- Data and Code Disclosure Policy: -- Reviewer guidelines: -- LaTeX style files: +- +- +- +- ## Submission Thesis -CRPTO should be read as data science for decisions: - -| IJDS component | CRPTO surface | +| IJDS dimension | CRPTO answer | |---|---| -| Data | Static Lending Club OOT panel, plus Prosper and Freddie/Mendeley frozen external stress tests. | -| Models/algorithms | Calibrated PD, Mondrian conformal intervals, robust LP, exact funded-set audit. | -| Decision relevance | Funding a credit portfolio under budget, risk tolerance, and uncertainty. | -| Implications | Model-risk governance, reproducible auditability, robust-price interpretation, and limits of external transfer. | +| Data | Temporal Lending Club panel with conformal fit, November selection, December audit, and OOT evaluation. | +| Method | Exact 90% conformal replay, deterministic endpoint cap, and one midpoint portfolio guardrail. | +| Decision | Allocate `$1M` under capital, concentration, and effective-PD constraints. | +| Evidence | Split nine-cell selector/audit, matched point-PD decision, temporal reversals, and month-cluster bootstrap. | +| Implication | An inspectable price of uncertainty, including cases where the static guardrail should be rejected. | + +## Completed Scientific Refactor + +- Retired approximate cross-alpha headline values. +- Replayed conformal quantiles exactly at every sensitivity alpha. +- Selected the conventional 90% reference level; documented endpoint + saturation at tighter levels. +- Replaced nonlinear/tail policy families with `q=(p+u)/2`. +- Separated point-PD economics from conformal feasibility. +- Reduced policy selection to a round-number `3x3` calibration grid. +- Replaced the Markov-based selector screen with deterministic `B_u<=0.28` and + documented the exact cap-stability interval. +- Isolated outcomes from a 12-column selector frame; November selects and an + outcome-free December replay checks policy identity. +- Added the independent December decision audit, including the funded-set + coverage miss, and a 31-month cluster bootstrap. +- Added matched point-PD and 75% blend comparators. +- Promoted temporal reversals and limitations to the body. +- Rebuilt A35--A40 and active claim-sync tests. -## Work Plan +## Remaining Submission Work -| Window | Goal | Required output | +| Window | Deliverable | Exit condition | |---|---|---| -| Jun 9-16 | Editorial contract | Body and supplement explicitly state data-model-decision-implication logic. | -| Jun 17-24 | Claim hardening | Every headline number maps to an artifact and a non-overclaim boundary. | -| Jun 25-Jul 2 | Related-work pressure test | Closest-work table reads as a novelty boundary, not a literature survey. | -| Jul 3-10 | Method and theorem audit | Exact funded-set certificate, weighted-validity assumption, and post-selection boundary are unambiguous. | -| Jul 11-17 | External replication polish | Prosper/Freddie remain evidence of recipe transfer, not new exact certificates. | -| Jul 18-24 | Figures and tables | Captions state takeaway; tables fit IJDS; figures remain readable in grayscale. | -| Jul 25-31 | Reproducibility package | Data/code disclosure plan, commands, hashes, DVC pointers, and raw-data instructions are ready. | -| Aug 1-5 | Official template | Body compiles in `informs4` with `dblanonrev`; current local build is 26 pages total, Section 9 and References start on page 22, so the body remains within the 25-page limit when references are excluded. Final ScholarOne proof still pending. | -| Aug 6-8 | Double-anonymous QA | Metadata, URLs, acknowledgements, local paths, and author signals are removed from reviewer-facing PDFs. | -| Aug 9-10 | Submission freeze | `just lint`, `just smoke`, `just validate-champion`, `just paper-submission-pdf`, and visual QA pass. | +| Jul 9--12 | Code and claim gates | Ruff, mypy, ty, focused tests, smoke, manifest, and drift gate green. | +| Jul 12--18 | PDF editorial QA | Official body and supplement render; no undefined citations; body within 25-page rule; visual QA complete. | +| Jul 18--24 | Reproducibility archive | Sanitized commands, source notes, run tags, hashes, and A35--A40 bundle staged. | +| Jul 25--31 | Anonymous package | Body, supplement, title page, cover letter, and disclosure form separated correctly. | +| Aug 1--8 | Cold review | Read only the generated PDFs; fix clarity, table, and citation defects. | +| Aug 9--10 | ScholarOne freeze | Upload, inspect ScholarOne proof, and submit only after go/no-go checklist. | -## The 15 Improvement Tracks +## Acceptance Risks -| # | Track | Done definition | -|---:|---|---| -| 1 | Central methodological claim | Abstract, introduction, and conclusion describe CRPTO as an auditable conformal-robust decision certificate. | -| 2 | IJDS fit | The body visibly contains data, method, decision, and implication components. | -| 3 | Exact-certificate language | "Exact" is defined as funded-set accounting on frozen OOT outputs, with statistical assumptions stated separately. | -| 4 | Finite-grid frontier | A35 is explained as the final evaluated pool93 finite-grid frontier, not all candidate policies or a continuous region. | -| 5 | External datasets | Prosper/Freddie are frozen external economic replications, not new Lending Club champions. | -| 6 | Related work | The closest-work boundary distinguishes CRPTO from P2P OR, conformal credit scoring, conformal RO, DFL, and financial portfolios. | -| 7 | Figures | Main figures have single-sentence takeaways, readable axes, grayscale-safe contrast, and no unnecessary decorative elements. | -| 8 | Tables | Body tables are compact reviewer evidence; voluminous diagnostics stay in the supplement. | -| 9 | Supplement | A3--A39 are organized as a defense layer with scope caveats. | -| 10 | Reproducibility | Accepted-paper package has code, DVC pointers, manifest, raw-data instructions, and guardrail commands. | -| 11 | Double anonymity | Reviewer-facing body and supplement contain no author URLs, names, local paths, or private remotes. | -| 12 | Official IJDS template | `CRPTO_ijds_submission.tex` is regenerated from the pool93 A35--A39 QMD source, compiles against the official files, and is rechecked after body edits. | -| 13 | Data/code form | Cover letter and disclosure text acknowledge IJDS accepted-paper reproducibility requirements. | -| 14 | Acceptance-risk audit | A short list of likely reviewer objections has body or supplement responses. | -| 15 | Freeze discipline | Protected champion/search stages are never rerun as routine paper reproduction. | +| Risk | Mitigation in current draft | +|---|---| +| Applied pipeline rather than method | One explicit objective/constraint contract and exact selector protocol. | +| Broad binary conformal intervals | A35 reports width and endpoint saturation; no 99% headline. | +| Adaptive funded-set validity | December directly demonstrates the coverage miss; deterministic accounting is separated from conditional Markov language. | +| Historical OOT reuse | "Retrospective lockbox replay" stated in abstract, design, limitations, supplement, and cover letter. | +| Baseline cherry-picking | Same candidates, budget, concentration, LGD, solver, and `tau`; temporal failures are shown. | +| Too many methods | A1--A34 demoted to diagnostics; A35--A40 support one midpoint policy. | +| Reproducibility mistaken for novelty | Decision method and managerial trade-off lead; tooling supports auditability. | +| Page and template risk | Official `informs4` build and visual QA are blocking gates. | -## Current Acceptance Risks +## Freeze Rule -| Risk | Why it matters | Mitigation | -|---|---|---| -| Perceived as applied pipeline | IJDS needs methodological data science, not just a case study. | Keep the decision-certificate framing central. | -| Overreading exact validity | Reviewers may object if "exact" sounds like universal conformal validity. | Define exact as funded-set accounting and state weighted validity separately. | -| External claims too strong | Prosper/Freddie are not new certificates. | Label them as economic replication and exhaustiveness audits. | -| Regret comparator confusion | SPO+ wins regret by design. | Present regret-auditability as a frontier with different governance outputs. | -| Template/page risk | Local HTML-print PDFs are not official. | Keep the `informs4` handoff build current and recheck the ScholarOne proof before submission. | -| Reproducibility policy | IJDS requires disclosure form at submission and archive workflow at acceptance. | Maintain `REPRODUCIBILITY_PACKAGE.md` and cover-letter language. | +After the scientific and PDF gates pass, do not reopen the policy for marginal +OOT gains. Reopen only for a concrete reviewer request, a simpler calibration- +only rule that matches the active result, or a formally stronger prospective or +selection-valid protocol. diff --git a/paper/submission/README.md b/paper/submission/README.md index d9d3017..7a9d1e5 100644 --- a/paper/submission/README.md +++ b/paper/submission/README.md @@ -1,207 +1,133 @@ # IJDS Submission Package -This directory is the handoff checklist for the IJDS submission surfaces. The -source of truth remains: - -- `paper/CRPTO_ijds.qmd` for the anonymous manuscript body. -- `paper/supplement_ijds.qmd` for the anonymous online supplement. -- `paper/submission/COVER_LETTER_AND_DISCLOSURE.md` for editor-facing cover - letter language and data/code disclosure timing. -- `paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md` for the internal - two-month readiness plan. -- `paper/submission/CLAIM_AUDIT_MATRIX.md` for the claim/evidence/risk map. -- `paper/submission/REPRODUCIBILITY_PACKAGE.md` for the IJDS data/code package - plan. -- `paper/submission/TITLE_PAGE_DRAFT.md` for the separate non-anonymous title - page. -- `paper/submission/DATA_CODE_DISCLOSURE_FORM_DRAFT.md` for official-form - answer drafting. -- `paper/submission/SCHOLARONE_FINAL_CHECKLIST.md` for the final upload/proof - checklist. - -## Render Commands +This directory contains the official-template handoff and editor-facing +submission materials. The synchronized scientific sources are: + +- `paper/CRPTO_ijds.qmd`: anonymous body. +- `paper/supplement_ijds.qmd`: anonymous online supplement. +- `paper/submission/CRPTO_ijds_submission.tex`: manually compacted + `informs4` handoff. +- `paper/submission/CLAIM_AUDIT_MATRIX.md`: active claim/evidence map. +- `paper/submission/REPRODUCIBILITY_PACKAGE.md`: data/code package plan. + +The active manuscript has one policy: exact 90% conformal replay, +`q=(p+u)/2`, `tau=0.17`, and a nine-cell November selector under +`B_u<=0.28`. An outcome-free December replay and post-selection audit are part +of A36; the audit deliberately records that stable policy identity does not +imply selected-set coverage. A35--A40 are the active evidence bundle. Keep +body, supplement, TeX, and governance numerically aligned with +`tests/test_ijds_active_claim_sync.py`. + +## Preview ```powershell +just ijds-evidence just paper-submission just paper-submission-pdf ``` -The HTML render is the writing preview. The PDF render is a local HTML-print -verification draft for pagination and visual inspection. The final submission -PDF should be produced with the official INFORMS IJDS LaTeX template and -double-anonymous review option (`dblanonrev`), not with a hand-written local -style. +HTML and browser-print PDFs are writing and visual-QA previews. The upload PDF +must come from `CRPTO_ijds_submission.tex` with the official INFORMS class and +the `dblanonrev` option. -Editorial submission notes, venue reminders and page-budget comments belong in -this README or the cover-letter checklist, not in the anonymous manuscript body. +## Official Sources -`COVER_LETTER_AND_DISCLOSURE.md` is intentionally separated from the anonymous -body and supplement. Use it for editor-facing disclosure fields, then keep the -reviewer packet free of public repository URLs, author identity, local paths and -private remote details. - -## Official Template Sources - -- INFORMS author portal: +- INFORMS style files: - IJDS submission guidelines: -- IJDS data/code disclosure policy: +- IJDS data/code policy: - IJDS reviewer guidelines: -- Overleaf template page: - -Do not vendor private template downloads, reviewer forms, or authenticated -publisher material into this repository. Local copies of the official template -files can live in this directory for compilation, but they are gitignored on -purpose. - -## Official LaTeX Submission Build - -`CRPTO_ijds_submission.tex` is the official-template handoff draft in the -INFORMS class (`\documentclass[ijds,dblanonrev]{informs4}`). The source of -truth remains `paper/CRPTO_ijds.qmd`; after the 2026-07-02 pool93 A35--A39 -update, the `.tex` must be regenerated before any ScholarOne freeze. The synchronized -submission surface should carry the central IJDS body: title, abstract, -keywords, core sections, the journal pipeline Figure 1, the bound-claim stack, -the A35 finite-grid frontier, the A36--A39 selected-allocation audits in the -supplement, the regret-auditability comparison, plus the core, exact-certificate, -funded-set audit and regret tables. The -`informs2014.bst` + `../../book/references.bib` bibliography wiring is already -present. Journal figures use PDF/vector exports from `reports/crpto/figures/` -where possible; Figure 1 intentionally uses the PNG export because the vector -PDF crop box cuts the right edge under `informs4`. - -> **`informs4` is not on CTAN/TeX Live.** The class and bibliography style are -> distributed through the INFORMS author portal or the IJDS Overleaf template. -> Local copies are allowed for compilation and are gitignored. Do not commit -> `informs4.cls`, `informs2014.bst`, template PDFs, `.sty` files, or generated -> LaTeX build artifacts. - -Current local build state (verified 2026-07-06): TinyTeX/TeX Live 2026 and the -`listingsutf8` TeX package compile `CRPTO_ijds_submission.tex` to a 26-page -official-template PDF. Section 9 (Conclusion) and References both start on page -22, so the body remains inside the IJDS 25-page initial-submission budget when -references are excluded. The only LaTeX log warnings left are a small -`\maketitle` overfull from the `informs4` anonymous title block and font-size / -underfull paragraph warnings, visually acceptable unless the final ScholarOne -proof shows a layout issue. - -To produce the official submission PDF: - -1. Download or refresh `informs4.cls` and `informs2014.bst` from the INFORMS - author portal (or Overleaf) and drop them next to - `CRPTO_ijds_submission.tex`. These are gitignored on purpose - (`paper/submission/.gitignore`); do not commit them. -2. Build with `latexmk` when the local TinyTeX wrapper works: - - ```powershell - latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex - ``` - - If PowerShell/TinyTeX fails with `runscript.tlu`/`nil`, use the proven - fallback: - - ```powershell - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - bibtex CRPTO_ijds_submission - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - ``` - -3. The `dblanonrev` option keeps the body anonymous; verify against the anonymity - checklist below before uploading. - -## Anonymity Checklist - -- Manuscript metadata uses `author: "Anonymous"`. -- Supplement metadata uses `author: "Anonymous"`. -- Public GitHub, DVC, MLflow, DagsHub and personal URLs are described as a - companion package but not exposed in the double-anonymous body. -- Cover-letter/data-code wording lives in `COVER_LETTER_AND_DISCLOSURE.md`. -- Title-page, acknowledgements and repository disclosure are kept for the cover - letter or for post-acceptance policy, not the anonymous manuscript. -- Local paths, private workspace paths and usernames do not appear in the submitted - sources. - -## SPO+ Numbering Rule - -Use the committed A19/Figure 15 artifact for body claims: - -- Two-stage regret: `0.425896` -- SPO+ regret: `0.216837` -- Relative reduction: `49.09%` -- Wilcoxon: `p = 1.39e-164` - -The PyEPO 1.3.7 closeout remains a curated appendix note: - -- Two-stage regret: `0.358073` -- SPO+ regret: `0.184366` -- Relative reduction: `48.51%` -- Wilcoxon: `p = 3.80e-163` - -These protocols are compatible but not interchangeable. - -## Final Assembly Checklist - -- Recheck the official IJDS sources linked above within the final submission - week; policies and forms can change. -- Verify the Data and Code Disclosure Form language against - `REPRODUCIBILITY_PACKAGE.md` and `DATA_CODE_DISCLOSURE_FORM_DRAFT.md`. -- Convert `TITLE_PAGE_DRAFT.md` into the separate title page requested by - ScholarOne. -- Use `SCHOLARONE_FINAL_CHECKLIST.md` while uploading and reviewing the generated - proof. -- Recheck the official-template page budget if the body changes materially. The - current local official-template build is 26 pages total; Section 9 and - References start on page 22, keeping the body within the 25-page limit when - references are excluded. -- Keep A3--A39 in the online supplement unless a reviewer-facing argument needs - one compact table in the body. -- Preserve CRPTO as the coverage/auditability method and SPO+ as the low-regret - comparator. -- Cross-check every headline claim against `CLAIM_AUDIT_MATRIX.md`. -- Keep `CRPTO_ijds_submission.tex` synchronized with the QMD whenever the body - adds or demotes a figure, table, theorem statement or major result paragraph. -- Regenerate previews with `just paper-submission-pdf` before release. -- Run the repository gates: `just lint`, `just smoke`, `just validate-champion`. - -## Final Step - Official Compile - -This remains a final-week action before upload, gated on an explicit decision -to freeze the manuscript. `CRPTO_ijds_submission.tex` already carries the full -ported prose, the economic-anchor ladder, and the temporal-split and tail-risk -tables. Local publisher class/style files are available in this directory and -gitignored; recheck the official source before final submission in case INFORMS -updates the template. - -1. **Confirm closure.** The body content, numbers, and figures are final and the - repository gates pass. -2. **Refresh the official template files if needed** (gitignored on purpose; - never commit): - - `informs4.cls` and `informs2014.bst` from the INFORMS author portal - or the - IJDS Overleaf template (v2.00, 29 Apr 2025, the latest checked locally) - . - - Drop both next to `CRPTO_ijds_submission.tex` if local copies are absent or - stale. The fastest fallback is to paste the `.tex` into the Overleaf - template, which already bundles both files. -3. **Compile:** - - ```powershell - latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex - ``` - - Use the documented `pdflatex -> bibtex -> pdflatex -> pdflatex` fallback if - the local TinyTeX wrapper fails. - - ```powershell - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - bibtex CRPTO_ijds_submission - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - ``` - -4. **Recount the official-template page budget** and demote body floats to the - supplement only if the body exceeds 25 pages excluding references. The local - official-template build is currently 26 pages total; Section 9 and References - start on page 22. The Chrome-print body preview is only a verification proxy. -5. **Verify anonymity** against the checklist above, then upload the body PDF and - submit the title page separately. +- IJDS Overleaf template: + +Local copies of `informs4.cls`, `informs2014.bst`, and related publisher files +are used for compilation and remain gitignored. Recheck official sources during +the final submission week. + +## Official Build + +Run from `paper/submission`: + +```powershell +if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } +latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex +``` + +`latexmk` is preferred because it automates convergence. The repository build +resolves TinyTeX's `latexmk.pl` and launches it with Perl on Windows, bypassing +the defective `runscript.tlu` executable wrapper. If that payload is unavailable +or fails, the robust fallback is: + +```text +pdflatex -> bibtex -> pdflatex -> pdflatex +``` + +```powershell +pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex +bibtex CRPTO_ijds_submission +pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex +pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex +``` + +The three `pdflatex` calls are intentional: + +1. The first pass writes `.aux`, including citation keys and unresolved labels. +2. BibTeX reads `.aux` and writes the formatted `.bbl`. +3. The second LaTeX pass imports `.bbl` and resolves citations and references. +4. The final pass stabilizes labels, float positions, and pagination changed by + the bibliography and cross-references. + +After a TeX Live update, an `expl3` format mismatch can be repaired once with: + +```powershell +if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } +fmtutil-sys --byfmt pdflatex +``` + +The repository wrapper runs the working `latexmk` payload first and falls back +automatically: + +```powershell +just paper-submission-official +``` + +## Build Acceptance + +The official-template PDF is acceptable only when: + +- `.blg` has no bibliography warnings; +- `.log` has no undefined citations or references; +- the body is at most 25 pages under the IJDS counting rule; +- tables do not overflow or become unreadably small; +- the PDF remains double-anonymous; +- visual inspection confirms that figures, equations, and references render. + +Current verified build (2026-07-09): 13 pages total; References begin on page +11, so the main text occupies pages 1--10 and remains well within the 25-page +limit. The bibliography is clean, and visual inspection of all 13 pages found +no clipping, overlap, missing glyphs, or unreadable tables. + +Do not keep a page-count statement in this README without rebuilding the +current TeX. The final compile wrapper records the current count and warning +scan. + +## Anonymity + +- QMD metadata uses `author: "Anonymous"`. +- TeX uses `\documentclass[ijds,dblanonrev]{informs4}`. +- Author names, acknowledgements, repository ownership, personal URLs, local + usernames, and private remotes stay out of the reviewer packet. +- Cover-letter and data/code language lives in + `COVER_LETTER_AND_DISCLOSURE.md` and related editor-facing files. +- `TITLE_PAGE_DRAFT.md` is uploaded separately when ScholarOne requests it. + +## Final Gate + +```powershell +just submission-check +uv run dvc status --no-updates +git status --short +``` + +Use `SCHOLARONE_FINAL_CHECKLIST.md` for upload order and proof review. Preserve +the active midpoint narrative; OCE/CVaR, SPO+, external replications, and other +historical diagnostics remain supplement context rather than additional active +methods. diff --git a/paper/submission/REPRODUCIBILITY_PACKAGE.md b/paper/submission/REPRODUCIBILITY_PACKAGE.md index a47f34b..b3570b0 100644 --- a/paper/submission/REPRODUCIBILITY_PACKAGE.md +++ b/paper/submission/REPRODUCIBILITY_PACKAGE.md @@ -1,143 +1,111 @@ # IJDS Reproducibility Package Plan -IJDS requires the Data and Code Disclosure Form at submission and expects -accepted computational papers to upload data/code and complete the journal's -reproducibility workflow unless an exemption applies. CRPTO can satisfy this -without exposing secrets, private credentials, local paths, or -author-identifying repository URLs during double-anonymous review. - Official policy: +CRPTO can support accepted-paper reproduction without exposing credentials, +local paths, or author identity during double-anonymous review. + ## Disclosure Timing -| Stage | What to disclose | What to withhold | +| Stage | Disclose | Withhold | |---|---|---| -| Initial double-anonymous submission | Neutral description of the companion package, data sources, DVC-style artifact validation, and code/data availability timing. | Public GitHub/DagsHub URLs, author identity, local paths, secrets, tokens. | -| If editors request verification during review | An anonymized archive or controlled access bundle with source, tables, figures, tests, and non-identifying artifact metadata. | Credentials, raw private data, non-anonymized remotes. | -| Acceptance | Public source repository, reproducibility commands, DVC pointers or downloadable processed artifacts where permitted, raw-data acquisition instructions, manifest hashes, and final rendered outputs. | Secrets and any data redistribution prohibited by source licenses. | +| Initial submission | Neutral package description, source-data availability, and release timing. | Repository ownership, personal URLs, local paths, secrets. | +| Editor-requested verification | Anonymized source, A35--A40, tests, and sanitized artifact metadata. | Credentials, private remotes, non-anonymous provenance. | +| Acceptance | Public source, environment lock, data instructions, artifact pointers/hashes, and final outputs. | Secrets and data prohibited from redistribution. | ## Package Contents -| Component | Files/directories | Purpose | +| Component | Files | Purpose | |---|---|---| -| Source code | `src/`, `scripts/`, `tests/`, `pyproject.toml`, `uv.lock`, `justfile`. | Rebuild tables, figures, journal package, and validation checks. | -| Manuscript | `paper/CRPTO_ijds.qmd`, `paper/supplement_ijds.qmd`, `paper/submission/`. | Reproduce body, supplement, and submission surfaces. | -| Frozen evidence | `EXTRACTION_MANIFEST.json`, `models/*.json`, `reports/crpto/tables/`, `reports/crpto/figures/`. | Tie claims to immutable metrics and rendered evidence. | -| Data pointers | `.dvc/`, `dvc.yaml`, `dvc.lock`, DVC remote notes. | Retrieve or verify processed artifacts outside Git. | -| Raw-data instructions | `RAW_DATA_SOURCE_NOTES.md`. | Let readers reconstruct inputs without committing raw CSVs or credentials. | -| Guardrails | `just smoke`, `just validate-champion`, publication target tests, DVC status. | Confirm reproducibility without rerunning protected search stages. | - -## Accepted-Paper Reproduction Commands +| Environment | `pyproject.toml`, `uv.lock`, `justfile`. | Recreate the Windows-first toolchain. | +| Method source | `src/models/conformal_alpha_grid.py`, `src/optimization/`, active experiment scripts. | Replay exact intervals and solve declared policies. | +| Active config | `configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml`. | Fix alpha, split 3x3 selector/audit, endpoint cap, and solver settings. | +| Active evidence | A35--A40 CSV/TeX files and `ijds_policy_governance.json`. | Tie every paper claim to generated evidence. | +| Manuscript | body QMD, supplement QMD, official TeX. | Reproduce reviewer-facing surfaces. | +| Data pointers | `dvc.yaml`, `dvc.lock`, `.dvc/`, raw-data notes. | Retrieve large artifacts where terms permit. | +| Guardrails | active claim sync, publication integrity, manifest regression. | Detect narrative or historical artifact drift. | + +## Active Artifact Contract + +| Evidence | Path | +|---|---| +| Exact alpha grid | `data/processed/experiments/champion_reopen//conformal/exact_alpha_grid.parquet` | +| Calibration selector | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_selection_grid.parquet` | +| Calibration audit grid | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_audit_grid.parquet` | +| Calibration decision audit | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_holdout_audit.csv` | +| OOT evaluation | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_oot_evaluation.csv` | +| Funded rows | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_full_oot_allocations.parquet` | +| Governance | `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` | +| Paper tables | `reports/crpto/tables/crpto_tableA35...A40_*` | + +Active run: +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. + +Exact-alpha run: +`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. + +The active policy evidence is intentionally separate from the manifest-protected +historical bundle. `tests/test_ijds_active_claim_sync.py` guards the submitted +claim; `tests/test_manifest_regression.py` guards frozen provenance. The +manifest is not rewritten to make a manuscript update look historical. + +## Reproduction Commands + +Paper-facing reproduction from frozen experiment outputs: ```powershell just setup-base -just smoke -just validate-champion -just tables -just figures -just evidence -just journal-package +just ijds-evidence +uv run pytest tests/test_ijds_active_claim_sync.py -q just paper-submission -just paper-submission-pdf +just paper-submission-official +just validate-champion ``` -Official IJDS-template PDF build, after `paper/submission/CRPTO_ijds_submission.tex` -is synchronized: +Full isolated methodology replay: ```powershell -cd paper/submission -latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex +just ijds-active-replay ``` -PowerShell/TinyTeX fallback proven in the local Codex environment: +The full replay recomputes the exact alpha grid, solves the nine policies on +November and December, opens December outcomes only for the independent audit, +evaluates the frozen selected policy, and rebuilds A35--A40. It writes only to +versioned experiment paths and does not overwrite the frozen PD model, +calibrator, historical intervals, or manifest. + +Official-template compilation is automated by: + +```powershell +just paper-submission-official +``` + +Manual Windows fallback: ```powershell cd paper/submission +if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex bibtex CRPTO_ijds_submission pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex ``` -Artifact-aware DVC verification, when credentials or public artifact access are -available: +## Data and Artifact Boundary -```powershell -uv run dvc status --no-updates -uv run dvc status -c -r dagshub -``` - -## Pool93 Body-Claim Artifacts +- Lending Club, Prosper, and Freddie/Mendeley raw data are distributed through + their original sources, not copied into Git. +- Large processed parquet and model binaries use DVC or journal-approved + artifact delivery when source terms permit. +- `EXTRACTION_MANIFEST.json` verifies the historical upstream bundle. +- Review-stage copies of DVC/configuration metadata must remove repository + ownership, remote URLs, credentials, and absolute local paths. +- If a remote is unavailable, provide journal-approved processed artifacts plus + hashes, schema/source notes, and the commands above. -The paper body point (A35 "Body/default balanced point") and its frontier come -from the pool93 champion-reopen experiments, generated *outside* the DVC DAG by -deterministic re-evaluation of a pre-declared finite policy grid over the -frozen Mondrian conformal intervals. The package therefore includes, verbatim -and frozen: +## Non-Routine Stages -| Artifact | Path | Role | -|---|---|---| -| A35 frontier | `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` (+ `.tex`) | Consolidated return-bound frontier; body point row. | -| A36 grade audit | `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv` (+ `.tex`) | Funded-set grade composition of the body allocation. | -| A37 tail risk | `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` (+ `.tex`) | LGD-grid repricing and CVaR/OCE diagnostics. | -| A38 cluster bounds | `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` (+ `.tex`) | Cluster-aware Hoeffding sensitivity. | -| A39 bootstrap | `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` (+ `.tex`) | 5,000-draw fixed-allocation bootstrap intervals. | -| Terminal claim governance | `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` | Declared return floor, claim hierarchy, do-not-claim list. | -| Consolidated governance | `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` | Authoritative body-point metrics and frontier counts. | - -Run tags: terminal `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal`; -consolidated `champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive`. -Their hashes are pinned in `EXTRACTION_MANIFEST.json` and enforced by -`tests/test_manifest_regression.py`; body-claim consistency with the manuscript -is enforced by `tests/test_pool93_body_claim_sync.py`. These artifacts are not -regenerated by `just tables`/`just figures` (which rebuild only the -rebaseline-chain tables); they ship as frozen evidence. - -Anonymity note: the governance JSONs embed absolute local paths in their -`source_paths`/`runtime_status` blocks. Copies shipped inside any -review-stage or accepted-paper archive must be path-sanitized (or the fields -dropped); the in-repo originals stay hash-frozen. - -## Hash and DVC Boundary - -- `EXTRACTION_MANIFEST.json` is the source of truth for protected artifact - hashes; `just validate-champion` runs the manifest regression tests. -- DVC metadata (`dvc.yaml`, `dvc.lock`, `.dvc/`) records large data/model - dependencies and outputs without placing raw CSVs, processed parquet files, or - model binaries in Git. -- The accepted-paper archive may include processed/model artifacts directly only - if the journal workflow and source-data terms permit it. Otherwise it should - include DVC pointers, source acquisition notes, and the commands above. -- Review-stage archives must sanitize author-local paths and repository remotes - if they are sent before acceptance. - -## Review-Stage DVC Sanitization Checklist - -Before sending any reviewer-facing archive or editor-requested verification -bundle, inspect and sanitize: - -- `.dvc/config`: remove or replace the configured `dagshub` remote name, - endpoint URL, account/repository slug, and any credential-dependent remote - settings. -- `dvc.lock`: keep stage dependency/output hashes, but inspect text fields for - author-local paths or non-anonymous remote references before packaging. -- Governance JSONs under `models/experiments/champion_reopen/`: drop or redact - `source_paths`, `runtime_status`, and any absolute `C:\Users\...` values in - the copy sent for review. Do not edit the in-repo originals because their - hashes are frozen. -- Submission docs: keep source instructions and DVC-style validation language, - but avoid public GitHub, DagsHub, MLflow, personal, affiliation, or local - machine URLs until the journal workflow allows disclosure. - -If the DVC remote is unavailable during acceptance checks, the contingency is to -provide journal-approved processed artifacts or model files, plus -`EXTRACTION_MANIFEST.json`, `dvc.lock`, raw-data acquisition notes, and the -paper-export commands above so reviewers can verify the paper-facing surfaces. - -## Non-Rerunnable Stages - -The following stages are not part of routine reproduction because they would -change the frozen champion or reopen the protected search: +The active reproduction does not run protected upstream stages: ```text crpto.pd.champion @@ -147,15 +115,15 @@ crpto.portfolio.optimization crpto.portfolio.bound_exact_eval ``` -Paper-facing exports are safe because they consume frozen inputs. Any protected -rerun requires a new branch, drift report, and explicit decision to create a new -run tag. +Those stages would retrain, rewrite frozen artifacts, or reopen historical +search. Any such run requires a distinct tag and drift report. -## Data-License Strategy +## Acceptance Checklist -The current plan is to publish code, derived tables/figures, manifest hashes, -and instructions for obtaining the raw public datasets. Processed artifacts and -models should be provided through DVC or a journal-approved supplement only when -source licenses and file sizes permit. If a dataset cannot be redistributed, the -package should include enough source instructions, schema notes, and commands for -a reader to rebuild comparable artifacts from legally obtained data. +1. Build in the locked `uv` environment. +2. Rebuild A35--A40 and run active claim sync. +3. Validate historical manifest hashes. +4. Compile and visually inspect the official PDF and supplement. +5. Sanitize author identity and local paths in the review archive. +6. Publish source-data acquisition instructions and artifact hashes. +7. Record any unavoidable platform-level numerical differences explicitly. diff --git a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md index 86a203a..d257269 100644 --- a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md +++ b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md @@ -1,82 +1,79 @@ # ScholarOne Final Checklist -Use this checklist only after the paper content is frozen and the official IJDS -template files have been downloaded outside Git. +Use only after the scientific content and official PDFs are frozen. -## Files to Prepare +## Files -| File | Source | Reviewer-facing? | Status | -|---|---|:---:|---| -| Anonymous manuscript PDF | `CRPTO_ijds_submission.tex` compiled with `informs4` and `dblanonrev`. | Yes | Source synchronized with pool93 A35--A39 and the dual-tag provenance passages; local official-template build verified 2026-07-06 (26 pages total; conclusion and references start on p. 22); final ScholarOne proof pending | -| Anonymous online supplement PDF | `paper/supplement_ijds.qmd` rendered and visually checked. | Yes | Local render and page QA pass; final ScholarOne proof pending | -| Separate title page | `TITLE_PAGE_DRAFT.md` converted into the ScholarOne/title-page format. | No | Pending ScholarOne copy | -| Data and Code Disclosure Form | Official IJDS form using `DATA_CODE_DISCLOSURE_FORM_DRAFT.md`. | Editor/system | Pending official form entry | -| Cover letter | `COVER_LETTER_AND_DISCLOSURE.md`, shortened if ScholarOne text boxes are tight. | Editor | Draft ready; final text-box copy pending | -| Optional reproducibility note | `REPRODUCIBILITY_PACKAGE.md` or excerpted text if requested. | Editor/system | Optional | +| File | Reviewer-facing | Local status | +|---|:---:|---| +| Anonymous manuscript PDF from `CRPTO_ijds_submission.tex` | Yes | Rebuild and recheck after every body edit. | +| Anonymous supplement PDF | Yes | Render and visually inspect. | +| Separate title page | No | Complete from `TITLE_PAGE_DRAFT.md`. | +| Data and Code Disclosure Form | Editor/system | Finalize from the draft. | +| Cover letter | Editor | Finalize from `COVER_LETTER_AND_DISCLOSURE.md`. | +| Reproducibility note/archive | Editor/system | Sanitize identity, paths, and remotes. | -## Official Template Build +## Official Build -1. Download or refresh `informs4.cls` and `informs2014.bst` from INFORMS/Overleaf. -2. Regenerate/synchronize `CRPTO_ijds_submission.tex` from the pool93 A35--A39 QMD source. -3. Place the template files next to `CRPTO_ijds_submission.tex`; local gitignored copies are already present. -4. Build with `latexmk` when the local TinyTeX wrapper works: - - ```powershell - latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex - ``` +```powershell +just paper-submission-official +``` - If PowerShell/TinyTeX fails with `runscript.tlu`/`nil`, use the proven - fallback: +The wrapper uses the direct `latexmk.pl` payload on Windows and falls back to the verified +`pdflatex -> bibtex -> pdflatex -> pdflatex` loop. Accept only when: - ```powershell - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - bibtex CRPTO_ijds_submission - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - ``` +- `.blg` has zero warnings; +- `.log` has no undefined citation/reference warnings; +- body page count satisfies the IJDS 25-page rule; +- figures and tables fit; +- PDF metadata and visible content remain anonymous. -5. Confirm body page count is at most 25 pages excluding references and - appendices. The local official-template build verified on 2026-07-06 is 26 - pages total; Section 9 (Conclusion) and References both start on page 22, so - the manuscript remains comfortably inside the IJDS page budget when - references are excluded. Recount after every official rebuild. +Current local build (2026-07-09): 13 pages total, with References beginning on +page 11; citation/reference scans are clean. Recount after every substantive +TeX edit. -## Final Local Gates +## Local Gates ```powershell +just ijds-evidence +uv run pytest tests/test_ijds_active_claim_sync.py -q +just publication-integrity just lint +just type-check +just type-advisory-full just smoke just validate-champion -uv run pytest tests/test_publication_targets.py -q +just paper-submission +just paper-submission-official uv run dvc status --no-updates -just paper-submission-pdf ``` +`dvc status` is a report, not permission to rerun protected stages. Do not +repair paper-stage drift by overwriting the frozen upstream chain. + ## Anonymous PDF QA -- Body PDF has no author names, affiliation, acknowledgements, public repo URLs, - DagsHub/MLflow URLs, local paths, or hidden PDF metadata that identifies the - author. -- Supplement PDF has no author names, affiliation, acknowledgements, public repo - URLs, DagsHub/MLflow URLs, local paths, or hidden PDF metadata. -- References render correctly with `informs2014.bst`. -- Figures are readable in grayscale or black-and-white printing. -- Tables do not overflow page margins. -- Captions state the takeaway and do not overclaim exact validity. -- Prosper/Freddie text remains labeled as external economic replication, not as - new exact funded-set certificates. - -## ScholarOne Proof QA - -ScholarOne generates a proof after upload. Before final submission: - -- **Go/no-go gate:** the ScholarOne-generated proof must be visually checked - before final submission. Broken proof, missing figures, wrong file order, - title-page leakage, or anonymous-body identity leakage means **NO-GO**: - retract, repair locally, regenerate, and upload again. -- Open the generated proof, not only the local PDF. -- Recheck title, abstract, keywords, file order, supplement designation, and - Data/Code Disclosure fields. -- Confirm the title page is not included in the anonymous reviewer PDF. -- Confirm the supplement is designated as online supplemental material. -- Confirm no optional file accidentally reveals identity to reviewers. +- No author names, affiliations, acknowledgements, repository ownership, + personal URLs, local usernames, or private remotes. +- Correct title, abstract, keywords, section order, and supplement designation. +- References use the official INFORMS bibliography style. +- No missing glyphs, clipped figures, overflow tables, orphan headings, or + unreadably small text. +- Active numbers match A35--A40 and governance. +- Temporal reversals and retrospective-design caveat remain visible. +- OCE/CVaR, SPO+, and external datasets remain diagnostics, not active methods. + +## ScholarOne Proof Go/No-Go + +Open the ScholarOne-generated proof, not only the local files. Submission is +**NO-GO** if any of these occur: + +- title page or author identity leaks into reviewer files; +- body/supplement order is wrong; +- figure, equation, table, or bibliography is missing or clipped; +- data/code answers differ from the cover letter; +- page count or anonymous-review option is wrong; +- uploaded PDF differs from the locally validated build. + +Repair locally, rerun the gates, re-upload, and inspect the new proof before +final submission. diff --git a/paper/submission/TITLE_PAGE_DRAFT.md b/paper/submission/TITLE_PAGE_DRAFT.md index 60f530b..eacede6 100644 --- a/paper/submission/TITLE_PAGE_DRAFT.md +++ b/paper/submission/TITLE_PAGE_DRAFT.md @@ -8,13 +8,12 @@ Upload it only as the separate title page requested by ScholarOne/INFORMS. ## Manuscript Title -CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit Portfolio -Decisions +CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios ## Keywords -conformal prediction; robust optimization; credit risk; predict-then-optimize; -portfolio decisions; reproducibility; model risk governance +conformal prediction; predict-then-optimize; credit risk; portfolio +optimization; calibration; reproducible data science ## Author diff --git a/paper/supplement_ijds.qmd b/paper/supplement_ijds.qmd index 4ea7355..35aebb3 100644 --- a/paper/supplement_ijds.qmd +++ b/paper/supplement_ijds.qmd @@ -1,5 +1,5 @@ --- -title: "Online Supplement for CRPTO" +title: "Online Supplement for CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios" author: "Anonymous" date: today lang: en @@ -22,945 +22,380 @@ execute: warning: false --- -::: {.callout-note} -## Scope - -This online supplement supports the IJDS submission body. It collects proof -details, robustness and external-replication tables A3--A39, reproducibility -commands, model-risk material, fairness diagnostics, and evidence lineage. It -does not introduce a hidden selection criterion or unreported claim family: -A35 is the declared finite-grid frontier consumed by the body, A36 -is the regenerated funded-set grade audit for the selected body point, -and A37--A39 add selected-policy tail-risk, cluster-bound, and fixed-allocation -bootstrap diagnostics from the same selected allocation. -::: - -The supplement is organized as a defense layer for the main manuscript rather -than as a second paper. Appendices A--C support the statistical and economic -claims, Appendix D records governance limits, Appendix E gives the reproduction -path, and Appendix F maps the anonymous submission files. Tables and figures are -included only when they either protect a body claim or prevent overreading of the -external and robustness evidence. - -::: {.callout-important} -## Journal Strengthening Pack - -Selected former P2/P3 ideas are included here when they can be evaluated from -frozen evidence: OCE/CVaR as a tail-risk diagnostic, robust satisficing as -committee-style margins, regret-auditability as the SPO+/CRPTO comparator, and -dependence-aware theory as a caveat/proposition. The multidataset layer is now -included as a frozen external economic replication on Prosper and Freddie/Mendeley: -it tests transfer of the CRPTO recipe without reopening the Lending Club -champion. Optimized OCE/CVaR objectives, full multi-distribution or online -conformal prediction, online DFL, causal CRPTO, multi-period portfolios, -production monitoring, and package extraction remain future work only. -::: - -# Appendix A: Theoretical Details - -The body states the CRPTO guarantee in operational terms. This appendix fixes -notation and records the exact boundary between the distribution-free claim and -the optional tightening arguments. The proof depends on Assumption 1 (weighted -funded-set validity) as a modeling premise for the selected allocation; Appendix -A does not turn the selected funded set into a universal conformal guarantee. - -| Symbol | Meaning | -|---|---| -| `Y_i` | Observed default indicator or bounded loss proxy for loan `i`. | -| `p_hat_i` | Calibrated probability of default used by the decision layer. | -| `u_i(alpha)` | Upper conformal endpoint at miscoverage level `alpha`. | -| `x_i` | Funding decision or allocation weight. | -| `a_i` | Loan amount or exposure. | -| `w_i` | Normalized funded-set weight, `x_i a_i / sum_j x_j a_j`. | -| `gamma` | Robust-policy blend parameter in the optimization rule, `gamma in [0,1]`. | -| `p_tilde_i(alpha,gamma)` | Blended PD used by the optimizer, `p_hat_i + gamma (u_i - p_hat_i)`. | -| `tau` | Risk-tolerance cap the optimizer applies to the blended PD, `sum_i w_i p_tilde_i <= tau`. | -| `Gamma_CP` | Portfolio-level conformal risk metric after allocation: `sum_i w_i (u_i - p_hat_i)`, with clipping at one on the PD scale. | -| `B_u(alpha)` | Weighted upper-endpoint budget of the funded set, `sum_i w_i u_i(alpha)`; equals `tau + (1-gamma) Gamma_CP` when the cap binds. | -| `V(alpha)` | Weighted funded-set miscoverage quantity. | - -Prose writes the conformal premium as $\Gamma_{\mathrm{CP}}$; tables sometimes -use the CSV label `Gamma_CP` so regenerated headers remain traceable. - -For a fixed allocation, conformal coverage controls the expected interval miss -indicator under the stated exchangeability design. CRPTO converts loan-level -misses into the funded-set quantity `V(alpha)` using normalized exposure -weights. The expectation in Assumption 1 is over the exchangeable -calibration/test draw, conditional on the frozen recipe, declared partitions, -and allocation rule. The portfolio-level theorem assumes weighted funded-set -validity, `E[V(alpha)] <= alpha`, rather than claiming that marginal split -conformal automatically controls every adaptively selected subportfolio. A -Markov-style argument then gives the main conservative portfolio bound used in -the exact alpha-safe check. - -## Proof of Theorem 1 - -The body states the result as Theorem 1 under Assumption 1 (weighted -funded-set validity). We record the full argument here. - -**Setting.** The allocation $x$ is fixed before OOT labels are revealed. -Funded-set weights satisfy $w_i \geq 0$ and $\sum_i w_i = 1$. Outcomes and -endpoints live on the PD scale, $Y_i \in [0,1]$ and $u_i(\alpha) \in [0,1]$, -with miscoverage indicators $Z_i(\alpha) = \mathbf{1}\{Y_i > u_i(\alpha)\}$ -and $V(\alpha) = \sum_i w_i Z_i(\alpha)$. Write -$B_u(\alpha) = \sum_i w_i u_i(\alpha)$ for the weighted upper-endpoint budget of -the funded set. The robust layer caps the $\gamma$-blended PD, -$\sum_i w_i \tilde p_i(\alpha,\gamma) \leq \tau$ with -$\tilde p_i = \hat p_i + \gamma(u_i(\alpha) - \hat p_i)$; the deterministic -identity below does not use this cap and holds for any allocation. Probabilities -and expectations are over the exchangeable calibration/test draw conditional on -the frozen recipe, partitions, and allocation rule. - -**Step 1 (pointwise domination).** For every funded loan, +# Scope and Evidence Map -$$ -Y_i \;\leq\; u_i(\alpha) + Z_i(\alpha). -$$ +This supplement supports one active method: an exact 90% conformal replay, +the midpoint decision score $q=(p+u)/2$, risk tolerance $\tau=0.17$, and a +nine-cell calibration selector. A35--A40 are the active evidence bundle. +Earlier A1--A34 analyses remain useful as model, robustness, comparison, or +external-transfer diagnostics, but none selects or redefines the submitted +policy. -If $Z_i(\alpha) = 0$ then $Y_i \leq u_i(\alpha)$ by definition of the -indicator. If $Z_i(\alpha) = 1$ then $Y_i \leq 1 \leq u_i(\alpha) + 1$ -because $u_i(\alpha) \geq 0$. Both cases give the displayed inequality. +| Evidence | Role in the submission | Claim boundary | +|---|---|---| +| A35 | Exact alpha replay, coverage, width, and endpoint saturation. | Other alpha rows are sensitivities, not additional policies. | +| A36 | All nine calibration selector cells. | No OOT outcome-derived ranking columns. | +| A37 | Full-OOT and fixed-policy temporal evaluation. | Retrospective stress evidence, not prospective performance. | +| A38 | Funded exposure and outcomes by letter grade. | Business composition, not legal fair-lending certification. | +| A39 | Fixed-allocation month-cluster bootstrap plus loan-level sensitivity. | Does not resample model, conformal recipe, selector, or solver. | +| A40 | Selected, more-conservative, and matched point-PD decisions. | Retrospective contrasts, not causal effects or universal dominance. | -**Step 2 (deterministic identity, Theorem 1(i)).** Taking the $w$-weighted -sum of Step 1, +The active governance file is +`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. +The final tagged ranking code does not read OOT defaults or returns. Earlier +project development did inspect the static OOT corpus, so the evaluation is a +retrospective lockbox replay rather than a pristine prospective trial. -$$ -\sum_i w_i Y_i - \;\leq\; \sum_i w_i u_i(\alpha) + \sum_i w_i Z_i(\alpha) - \;=\; B_u(\alpha) + V(\alpha). -$$ +# Appendix A: Exact Conformal and Accounting Details -This step uses no probability and no optimality: it is portfolio accounting, -valid for every realization, and it is exactly the identity verified by the -frozen exact funded-set audit. +## A.1 Frozen Recipe Replay -**Step 3 (Markov, Theorem 1(ii)).** $V(\alpha)$ is a nonnegative random -variable under the stated draw, and Assumption 1 gives -$E[V(\alpha)] \leq \alpha$. Markov's -inequality yields, for every $t > 0$ [@ghosh2002], +Let $p_j$ be calibrated PD and $Y_j\in\{0,1\}$ the calibration outcome. The +active recipe uses the scaled nonconformity score $$ -P\bigl(V(\alpha) \geq t\bigr) \;\leq\; \frac{E[V(\alpha)]}{t} - \;\leq\; \frac{\alpha}{t}. +s_j=\frac{|Y_j-p_j|}{\sigma(p_j)},\qquad +\sigma(p)=\sqrt{\max\{p(1-p),10^{-6}\}}. $$ -**Step 4 (combination).** By Step 2, the event -$\{\sum_i w_i Y_i \geq B_u(\alpha) + t\}$ implies $\{V(\alpha) \geq t\}$, so +Calibration scores are partitioned into five score-quantile Mondrian cells. +For a cell with $n_g$ calibration rows, the finite-sample quantile level is $$ -P\!\left(\sum_i w_i Y_i \geq B_u(\alpha) + t\right) \;\leq\; \frac{\alpha}{t}, +\min\left\{1, +\frac{\lceil(n_g+1)(1-\widetilde\alpha)\rceil}{n_g} +\right\}, $$ -and the choice $t = \sqrt{\alpha}$ gives the body statement: -the endpoint budget $B_u(\alpha)$ is exceeded by more than $\sqrt{\alpha}$ with -probability at most $\sqrt{\alpha}$. $\blacksquare$ - -**Policy-term decomposition.** The optimizer constrains the $\gamma$-blended PD, -not $B_u(\alpha)$ itself. Since -$\sum_i w_i \tilde p_i(\alpha,\gamma) = \sum_i w_i \hat p_i + \gamma\,\Gamma_{\mathrm{CP}}(\alpha)$ -and $B_u(\alpha) = \sum_i w_i \hat p_i + \Gamma_{\mathrm{CP}}(\alpha)$, with -$\Gamma_{\mathrm{CP}}(\alpha) = \sum_i w_i\bigl(u_i(\alpha) - \hat p_i\bigr)$, +with `higher` interpolation and frozen used alpha +$\widetilde\alpha=0.095$ for target $\alpha=0.10$. Sparse cells fall back to +the recorded global recipe, and frozen group/temporal factors may widen but +never narrow an interval. For evaluation loan $i$, $$ -B_u(\alpha) = \sum_i w_i \tilde p_i(\alpha,\gamma) + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha) - \;\leq\; \tau + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha), +[\ell_i,u_i] +=\left[ +\max\{0,p_i-\widehat q_{g(i)}\sigma(p_i)\}, +\min\{1,p_i+\widehat q_{g(i)}\sigma(p_i)\} +\right] $$ -with equality when the cap binds. The selected body point binds at -$\tau = 0.1715$ with $\gamma = 0.5475$ and -$\Gamma_{\mathrm{CP}}(0.01) = 0.162616$, so -$B_u(0.01) \leq 0.245084$ and the deterministic bound reads -$\sum_i w_i Y_i \leq 0.245084 + V(0.01) = 0.280434$. The exact audit reports -zero deterministic violation. The slack $(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ is -the part of the conformal premium the optimizer does not internalize at -$\gamma < 1$; setting $\gamma = 1$ recovers the tight endpoint cap -$B_u(\alpha) \leq \tau$. - -On the promoted draw the realized weighted miscoverage is $V(0.01) = 0.035350$, -which *exceeds* the nominal $\alpha = 0.01$: the funded set under-covers relative -to the marginal conformal level (unweighted funded coverage `0.9427` at -alpha01 for the body point), as expected -for an adaptively selected subportfolio. The deterministic identity needs no -assumption and holds with wide margin; the probabilistic guarantee is therefore -read at the Markov level $V \leq \sqrt{\alpha} = 0.10$, which the realized $V$ -clears, not at the nominal $\alpha$. This is why Assumption 1 is presented as a -modeling premise audited after selection, not as a property the single frozen -draw establishes. The under-coverage is structural, not a calibration-draw -effect: with $n_{\mathrm{cal}} = 237{,}584$ calibration loans, the split-conformal -conditional-coverage result [@vovk2005; @angelopoulos2023] makes the marginal -coverage Beta-distributed about $1 - \alpha$ with standard deviation -$\sqrt{\alpha(1-\alpha)/n_{\mathrm{cal}}} \approx 0.0002$ at $\alpha = 0.01$, so the -endpoints (hence $\Gamma_{\mathrm{CP}}$ and $B_u$) are essentially invariant to the -calibration draw and the residual $V$ variability is test-side and -portfolio-selection driven. A36 regenerates the row-level funded-set grade audit -from the selected allocation. A37 reprices the same selected allocation -under LGD-sensitive CVaR/OCE tail summaries, and A38 recomputes the -concentration-sensitive cluster-bound thresholds from the selected funded -weights. A39 bootstraps funded-loan contributions under the fixed final -allocation. This is an empirical interval diagnostic, not a conformal guarantee, -and it does not resample solver inputs, the PD model, calibration data, or the -policy search. - -These steps leave a clean split. Theorem 1 proves the operational bound under -Assumption 1. Proposition A.1 below shows that Markov is the sharp -first-moment statement when no additional structure is asserted. Proposition -A.2 asks the next natural question: if the funded set is grouped by period, -grade, or period-grade, and dependence is allowed inside each group, what would -independence across groups buy? That cross-cluster assumption is plausible only -as an additional sensitivity condition, not as a theorem premise quietly added -to the body. - -The phrase "exact funded-set certificate" has a narrow meaning throughout the -paper. It is an exact accounting audit on the frozen out-of-time funded set: -given the selected allocation weights, observed outcomes, calibrated PD values, -and conformal upper endpoints, the audit computes `V(alpha)`, -`Gamma_CP(alpha)`, and violation directly on the funded loans. It is not a new -distribution-free theorem for arbitrary adaptive portfolios and it is not an -external-dataset certificate. Its statistical interpretation still requires the -weighted funded-set validity assumption in the body; its value is that the -promoted decision is not supported by a proxy metric, a graph, or a manually -transcribed table. - -| Certificate object | Computed from | What it supports | What it does not support | -|---|---|---|---| -| Exact funded-set audit | Frozen Lending Club OOT funded loans, allocations, labels, `p_hat`, and `u_i(alpha)`. | Body point `V(0.01)=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084`, zero violation, and `8/8` alpha-grid pass. | Universal conditional coverage or live adaptive control. | -| Finite-grid frontier | 51,678 raw rows consolidated into 50,010 deduplicated semantic policies. | 27,508 all-alpha above-floor policies; terminal endpoint search `37,068/37,068` all-alpha passers. | A claim about all continuous policy values or future searches. | -| External exhaustiveness audit | Prosper all-candidate and Freddie capped/all-candidate LP solves. | The reported external LP values are not shortlist effects. | New exact funded-set certificates for Prosper or Freddie. | - -: Certificate taxonomy used in the paper. - -## Finite-Grid Frontier Closure (A35) - -The frontier closure replaces single-policy reporting with a declared finite-grid -frontier. The declared return floor `$170,464.54` is the realized return of the -previously certified bound-aware allocation on the frozen upstream chain; the -frontier re-evaluates a pre-declared finite policy grid deterministically over -the *same* frozen PD model outputs and Mondrian conformal intervals, so promoting -the body point regenerates no upstream model, calibrator, or interval. The consolidated frontier file -deduplicates 51,678 raw rows into 50,010 -semantic policies; 27,508 policies both pass the declared alpha grid and exceed -the return floor. The terminal endpoint run evaluates 37,068 policies and -296,544 exact alpha checks, with 37,068/37,068 all-alpha passers. The alpha grid -is fixed at -`{0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}`. - -| Role | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---:|---:|---:|---:|:---:| -| Minimum Markov-cap endpoint | `$170,467.27` | `0.095719` | `0.031875` | `0.273036` | `8/8` | -| Low-cap balanced endpoint | `$171,006.20` | `0.097190` | `0.031875` | `0.274789` | `8/8` | -| Highest return under cap <= `0.30` | `$173,314.04` | `0.115400` | `0.035875` | `0.294580` | `8/8` | -| Strict cap <= `0.345` body proxy | `$184,800.41` | `0.162562` | `0.035350` | `0.344996` | `8/8` | -| Body/default balanced point | `$184,832.48` | `0.162616` | `0.035350` | `0.345084` | `8/8` | -| Highest return under cap <= `0.36` | `$186,050.73` | `0.174600` | `0.037750` | `0.358685` | `8/8` | -| Max-return economic endpoint | `$223,458.14` | `0.457438` | `0.069575` | `0.510753` | `8/8` | - -: Finite-grid return-bound frontier (A35). Source file: -`reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv`. - -## Sharpness of the Distribution-Free Bound - -The body keeps Markov as the headline because it is the distribution-free -first-moment statement under Assumption 1. Earlier concentration tables remain -useful as a diagnostic recipe, but they depend on the funded-loan exposure -weights of the selected allocation. After the selected-policy promotion, the paper-facing -certificate is therefore A35 plus the exact funded-set audit, with A38 providing -the regenerated cluster-bound sensitivity and A39 closing the -fixed-allocation bootstrap diagnostic. A38 confirms that the -period-grade partition has the smallest cluster-aware Hoeffding threshold among -the reported partitions, but it is still looser than Markov at the paper -threshold. The body therefore keeps the first-moment Markov statement rather -than adding unverified cross-cluster independence to the theorem. - -Table A21c extends the same audit to one-sided Cantelli, Bennett and Freedman -variants [@cantelli1929; @bennett1962; @freedman1975]. At `alpha = 0.01`, -weak-variance Cantelli gives `t = 0.0661`, Bennett gives `t = 0.0722`, and -Bernstein/Freedman give `t = 0.0852` under the weaker weighted-validity variance -proxy; the stronger individual-alpha variance mode is sharper but requires an -assumption we do not assert in the theorem. - -The menu also contains the row that justifies the body's choice, which we -state as a formal optimality result rather than a stylistic preference. - -**Proposition A.1 (Markov-optimality under Assumption 1 alone).** -Let $V(\alpha) \in [0,1]$ satisfy only Assumption 1, $E[V(\alpha)] \le \alpha$, -with no further independence, correlation, or variance structure. Then for the -one-sided deviation event $\{V(\alpha) \ge t\}$ at the interpretability level -$\delta = \sqrt{\alpha}$, Markov's threshold $t_{\mathrm M} = \sqrt{\alpha}$ -cannot be improved by any second-moment argument: the variance compatible with -Assumption 1 is at most $\sigma^2 \le \alpha(1-\alpha)$ (attained by a -Bernoulli$(\alpha)$ miscoverage profile), and the sharp one-sided Chebyshev -(Cantelli) threshold under that worst-case variance is -$t_{\mathrm C} = \alpha + 3\sigma = \alpha + 3\sqrt{\alpha(1-\alpha)} > \sqrt{\alpha}$ -for $\alpha \le 0.01$ (numerically $t_{\mathrm C}=0.3085 > 0.1000 = t_{\mathrm M}$). - -*Proof.* Cantelli gives $P(V \ge E[V] + k) \le \sigma^2/(\sigma^2 + k^2)$. -Setting the right side to $\delta = \sqrt{\alpha}$ and $E[V] = \alpha$ yields -$k = \sigma\sqrt{(1-\delta)/\delta}$, so -$t_{\mathrm C} = \alpha + \sigma\sqrt{(1-\sqrt\alpha)/\sqrt\alpha}$. At -$\alpha = 0.01$ this is $\sigma\sqrt{9} = 3\sigma$ above $\alpha$, i.e. -$0.01 + 3(0.0995) = 0.3085$, which exceeds $t_{\mathrm M}=\sqrt{0.01}=0.1$. -$\blacksquare$ - -The reading is that Markov is **not** a conservative placeholder in the body; -it is the optimal first-moment statement for the guarantee Theorem 1 actually -asserts. Every sharper row in the menu prices a specific additional assumption -(loan independence, conditional variance, or a martingale protocol), and the -selected body point clears the stated Markov check with -`V = 0.035350 <= 0.100000`. - -Bennett is the closest match to the finite funded-set calculation because it -was designed for independent, non-identically distributed summands using only -the variance of the sum and component bounds. Freedman is included only as the -martingale analogue of Bernstein: it would become relevant under a pre-declared -sequential validation protocol with bounded increments and a -conditional-variance process, which is stronger than the frozen replay used -here. - -The table's role is assumption pricing. The sharper rows show what a reviewer -would gain by accepting independence, variance, or martingale structure; they -are not promoted because the body theorem asserts only Assumption 1. Chebyshev -is omitted because one-sided Cantelli dominates it for this event; Azuma is -omitted because it duplicates Hoeffding numerically while adding a sequential -protocol assumption; Chernoff is omitted because its sharp threshold requires -individual miss probabilities bounded by `alpha`; and a naive union-Markov -correction over the finite policy frontier is vacuous at the paper alphas. The -tables are regenerated by -`scripts/build_concentration_bound_table.py` and -`scripts/build_bound_tightening_audit.py` from frozen funded-set weights. - -## Cluster-Aware Conditional Tightening - -Let clusters $g = 1,\ldots,G$ represent period, grade, or period-grade cells, -and define +before the recorded widening factors. + +The replay reads all settings from the frozen conformal result payload. At the +90% reference level, maximum absolute differences against the stored artifact +are `4.44e-16` for point predictions, `3.33e-16` for lower endpoints, and +`6.67e-16` for upper endpoints. The active claim therefore uses recomputed +finite-sample quantiles rather than average-width scaling. + +## A.2 Policy Algebra + +For a generic linear blend +$q_i=p_i+\gamma(u_i-p_i)$ and funded-exposure weights $w_i$, $$ -Z_g(\alpha)=\sum_{i\in g} w_i\mathbf{1}\{Y_i>u_i(\alpha)\},\qquad -W_g=\sum_{i\in g}w_i . +\Gamma_{\mathrm{CP}}=\sum_iw_i(u_i-p_i), $$ -Within each cluster, defaults and conformal misses may be arbitrarily -dependent. The useful structure, if one is willing to assert it, is -cross-cluster independence after conditioning on the calibration sample and the -fixed funded allocation. Among the three reported partitions, period-grade is -the most defensible compromise for a temporal credit panel: it separates -calendar cohorts while conditioning on risk grade. Period alone ignores grade -mix, and grade alone cuts across calendar dependence. - -**Proposition A.2 (cluster-aware Hoeffding under cross-cluster independence).** -Let $\mathcal F$ contain the calibration sample, the frozen conformal recipe, -the declared cluster partition, and the selected funded allocation. Suppose -that, conditional on $\mathcal F$, the cluster aggregates -$Z_1(\alpha),\ldots,Z_G(\alpha)$ are independent, satisfy -$0\le Z_g(\alpha)\le W_g$, and obey conditional weighted validity -$\sum_g E[Z_g(\alpha)\mid\mathcal F]\le\alpha$ (for example, it is sufficient -that $E[Z_g(\alpha)\mid\mathcal F]\le\alpha W_g$ for every cluster). Then, for -every $\delta\in(0,1)$, - $$ -P\!\left( - V(\alpha)\ge - \alpha + \sqrt{\frac{1}{2}\left(\sum_g W_g^2\right)\log\frac{1}{\delta}} - \;\middle|\;\mathcal F -\right)\le\delta . +\Gamma_{\mathrm{int}}=\gamma\Gamma_{\mathrm{CP}},\qquad +\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}. $$ -*Proof.* Let $\mu=\sum_g E[Z_g(\alpha)\mid\mathcal F]\le\alpha$ and -$S_2=\sum_g W_g^2$. Hoeffding's inequality for independent bounded summands -gives +At $\gamma=0.5$, the conformal premium is split equally. The endpoint budget is $$ -P\{V(\alpha)-\mu\ge s\mid\mathcal F\}\le \exp(-2s^2/S_2). +B_u=\sum_iw_iu_i +=\sum_iw_iq_i+\Gamma_{\mathrm{res}}. $$ -Taking $s=\sqrt{S_2\log(1/\delta)/2}$ and using $\mu\le\alpha$ gives the -displayed bound. Integrating over $\mathcal F$ gives the same unconditional -statement. $\blacksquare$ - -Proposition A.2 is therefore the natural complement to Proposition A.1. Under -Assumption 1 alone, A.1 shows why Markov is the sharp distribution-free claim; -under an explicit cross-cluster structure, A.2 shows exactly when a -Hoeffding-style tightening becomes available [@hoeffding1963; -@boucheron2013concentration]. At the paper level $\alpha=0.01$ with matched -tail probability $\delta=\sqrt{\alpha}=0.10$, the cluster-aware threshold is -tighter than Markov only if $\sum_g W_g^2<0.0070$. The frozen funded set is much -more concentrated: period, grade, and period-grade partitions have -$\sum_g W_g^2=0.2407$, $0.3572$, and $0.0914$, respectively, so the corresponding -thresholds are `0.5365`, `0.6512`, and `0.3344`, all looser than Markov's -`0.1000`. This proposition does not replace the main theorem; it names the -extra structure a reviewer would have to accept and makes the empirical -concentration cost transparent in A21. - -# Appendix B: P1 Evidence - -The first evidence block asks whether the selected policy is a fragile point -estimate. Tables A3--A11 are generated from frozen or derived evidence files and -should be read as post-selection audit evidence rather than as a new selection -protocol. - -| Table | Role | Current-paper use | -|---|---|---| -| A3 nested holdout | Documents the 5K -> 25K -> 276K validation chain. | Appendix evidence against post-selection overclaiming. | -| A4 segment-period sensitivity | Checks coverage and funded-set quantities by period and grade. | Appendix robustness. | -| A5 decision-aware selector | Summarizes the CROMS-style screen across conformal candidates. | Method-defense appendix, not new training. | -| A6 synthetic shift | Stresses coverage under controlled covariate perturbations. | Robustness appendix. | -| A7 funded-set loan export | Provides row-level funded-set auditability. | Supplement only. | -| A8 funded-set composition | Summarizes grade, amount, and risk mix. | Appendix robustness. | -| A9 strict temporal holdout | Separates confirmation evidence from earlier selection decisions. | Strong appendix evidence. | -| A10 finalist exact bound evaluation | Shows exact alpha checks for conformal finalists. | Defends rank-1 selection. | -| A11 enhanced synthetic shift | Extends stress around the promoted policy. | Robustness appendix. | - -These tables support the current claim that the promoted decision is not a -fragile single point. They do not turn the current paper into a new -prospective selection protocol. A fully pre-declared prospective protocol is -future journal hardening. - -# Appendix C: Journal Robustness Package - -Tables A12--A39 answer likely reviewer questions that are too detailed for the -25-page body. The block structure is deliberate: A12--A18 report tail, margin -and policy-family diagnostics; A19 isolates regret-auditability; A20--A24 stress -tail, dependence, multi-distribution and online interpretations; A25--A34 test -external recipe transfer; and A35--A39 close the selected-policy frontier, -composition, tail, concentration and uncertainty profile. A35 is the active -frontier consumed by the body, A36 is the selected-policy funded-set -grade audit, A37 is the selected-policy tail-risk repricing, A38 is the -selected-policy cluster-bound audit, and A39 is the selected-policy -fixed-allocation bootstrap audit. A12--A34 are diagnostic by design, except A19 -also supports the body framing around regret-auditability and A25--A34 support -the external-replication defense. -A20--A34 are journal-only add-ons: A20 audits tail-risk trade-offs as a -diagnostic package; A21 makes the dependence-aware caveat numerical; A22 turns -the tail-risk diagnostic into an *active CVaR/OCE selection constraint* (a -labeled challenger, not a new promoted policy); A23--A24 stress -multi-distribution coverage and online (ACI) stability on the frozen conformal -intervals; A25--A34 cover the external economic layer by testing the same frozen -recipe and adding the cross-dataset price-of-robustness scaling readout. Table -A12 follows the standard -definitions of Conditional -Value-at-Risk [@rockafellar2000cvar] and the Optimized Certainty Equivalent -[@bental2007oce], and the non-monotonic risk-control framing behind A22 follows -[@angelopoulos2026nonmonotonic]; all are reported as post-hoc summaries of the -frozen funded set or intervals, never as a re-promoted champion. - -| Table | Role | Scope caveat | -|---|---|---| -| A12 tail-risk OCE/CVaR diagnostics | Reprices the funded set under tail-risk summaries. | Diagnostic only; OCE/CVaR is not the optimized objective. | -| A13 satisficing margins | Expresses return, `V`, `Gamma_CP`, violation, and frontier pass as margins. | Thresholds are explanatory, not a new policy selector. | -| A14 dependency cluster diagnostics | Documents period/grade concentration for the tightening caveat. | Does not prove independence. | -| A15 leave-one-period stress | Reweights the funded set by leaving periods out. | Descriptive stress, not re-optimization. | -| A16 bootstrap funded-set metrics | Adds empirical intervals for return, defaults, `V`, and misses. | Bootstrap interval, not conformal guarantee. | -| A17 budget/LGD/cap sensitivity | Varies operating assumptions. | Segment caps are diagnostics, not solver constraints. | -| A18 policy-family robustness surface | Summarizes alpha-safe policies by bound-aware family. | Compatible leaderboard only inside the final family; A35 is the selected frontier. | -| A19 regret-auditability frontier | Compares two-stage, SPO+, and CRPTO robust on regret versus risk-control checks. | Comparator framing, not a new champion selector. | -| A20 tail-risk diagnostic audit | Ranks tail-risk alternatives by CVaR, OCE, return, and satisficing status on the legacy diagnostic surface. | Journal diagnostic machinery; A37 is the selected-allocation tail repricing. | -| A21 cluster-bound tightening | Reports cluster-aware Hoeffding thresholds by period, grade, and period-grade. | Transparent caveat; not tighter than Markov for the observed exposure concentration. | -| A22 tail-constrained re-optimization | Selects the max-return policy under a decision-time CVaR cap computed from conformal upper endpoints, tracing the return-vs-CVaR frontier. | CVaR/OCE as an active selection constraint; reports a tail-constrained challenger, not a new promoted policy. | -| A23 multi-distribution robustness | Worst-case 90% coverage by grade and grade x vintage cell on the frozen intervals. | Read-only diagnostic; the worst fine cell motivates MDCP/group-weighted as future work. | -| A24 online conformal stability | Per-vintage and cumulative coverage plus the Gibbs-Candes ACI target trajectory over the OOT vintage sequence. | Static-OOT online-control diagnostic, not a streaming validation. | -| A25 external replication gate | Applies the frozen CRPTO scoring/conformal/LP recipe to Prosper final-status loans and Freddie/Mendeley FM48. | External economic replication; not a Lending Club champion rerun and not a new exact theorem. | -| A26 external candidate sensitivity | Checks whether the robust LP objective is stable as the OOT candidate pool grows. | Candidate-pool audit; supports the claim that reported objectives are not a tiny shortlist effect. | -| A27 Freddie horizon sensitivity | Audits Freddie/Mendeley default windows and red/green groups before selecting FM48 for the external table. | Dataset-selection audit; FM48 is promoted because it clears both coverage gates and keeps positive robust LP value. | -| A28 external LP exhaustiveness | Solves Prosper all-candidate LP and Freddie caps `500k`, `1M`, and `all`. | Exhaustiveness audit; the Freddie all-candidate optimum matches the screened optimum. | -| A29 Freddie sparse Mondrian audit | Splits Freddie coverage by all groups, eligible groups, and sparse fallback groups. | Documents sparse cells; does not claim perfect conditional coverage in every tiny group. | -| A30 external metric intervals | Adds uncertainty intervals for AUC, coverage, alpha coverage, and robust objective. | Bootstrap for funded-loan contribution only; it does not resample solver inputs. | -| A31 external OOT subperiod metrics | Breaks Prosper by OOT year and Freddie by OOT quarter. | Subperiod audit; Freddie 2015Q4 alpha coverage is just below 99%. | -| A32 Prosper default-definition sensitivity | Repeats Prosper under main, defaulted-only, and chargedoff-only labels. | Default semantics audit; all three variants pass the global gates. | -| A33 Freddie segment sensitivity | Repeats Freddie FM48 for red, green, and combined groups. | Segment audit; green and combined pass alpha01, red remains a documented caveat. | -| A34 cross-dataset price of robustness | Orders the frozen external applications (Freddie green/combined/red and Prosper) by panel default rate and reports the signed price of robustness. | Positive premium under blind external application, ordered by panel default risk across four frozen external applications. | -| A35 finite-grid frontier | Declares the promoted return-bound frontier and selected body point. | Exact finite-grid policy surface, not a continuous-region optimum. | -| A36 funded-set grade audit | Regenerates the selected body allocation at `alpha = 0.01` and summarizes funded exposure by grade bucket. | Composition evidence for the selected body point, not a fairness or protected-class audit. | -| A37 selected-policy tail-risk repricing | Reprices the selected body allocation under LGD alternatives, CVaR, OCE, and decision-time tail loss. | Diagnostic risk profile of the selected point; OCE/CVaR is not the optimized objective. | -| A38 selected-policy cluster-bound audit | Recomputes period, grade, period-grade, and score-vintage concentration thresholds from selected funded weights. | Sensitivity under extra cross-cluster assumptions; Markov remains the body-level bound. | -| A39 fixed-allocation bootstrap audit | Bootstraps funded-loan contributions under the selected body allocation. | Empirical contribution interval only; solver inputs, model, calibration, and policy search are not resampled. | - -## External Multi-Dataset Replication - -The external layer addresses the most natural single-dataset criticism without -changing the Lending Club frontier claim. Prosper final-status loans provide a -second P2P-style consumer-credit panel with direct investment fields -[@prosperLoanData]. Freddie/Mendeley FM48 provides a mortgage-credit panel built -from Freddie Mac loan-level performance data with train/OOS/OOT splits and -multiple default windows [@freddieMacSfLoanLevel; @mushava2023classimbalance]. -Home Credit was audited as a scoring/conformal source [@homeCreditDefaultRisk] -but is not promoted because it lacks a clean `exposure + return` investment -contract comparable to Lending Club, Prosper, or Freddie. - -| Dataset | Rows | Default | AUC | Cov. 90% | Cov. alpha01 | OOT cand. | Robust LP | -|---|---:|---:|---:|---:|---:|---:|---:| -| Prosper final-status | `54,807` | `30.92%` | `0.7074` | `0.9205` | `0.9943` | `10,531` | `$199,419` | -| Freddie/Mendeley FM48 | `3,173,355` | `1.45%` | `0.7839` | `0.9745` | `0.9907` | `1,396,053` | `$1,291,228` | - -: A25. External replication gate on the two reported economic datasets. The -source CSV is `reports/crpto/tables/crpto_tableA25_external_replication_gate.csv`. - -![A25 answers whether the recipe travels beyond Lending Club: Prosper and -Freddie/Mendeley both clear the global conformal and alpha = 0.01 coverage checks, with positive robust LP -objectives.](../reports/crpto/figures/crpto_fig22_external_replication.png){#fig-supp-external-replication width="88%" fig-alt="Bar chart comparing Prosper final-status and Freddie FM48 on coverage 90 percent, alpha 0.01 coverage, AUC and robust LP objective."} - -Table A26 checks whether the reported external robust objectives depend on an -artificially tiny candidate pool. Prosper reaches the same robust LP objective -using all `10,531` OOT candidates. Freddie remains stable at `$1,291,228` from -the top-`50,000` through top-`250,000` candidate pools, while random pools improve -monotonically with larger caps and stay below the top-return screen. A28 then -removes the remaining shortlist concern by solving Freddie on `500,000`, -`1,000,000`, and all `1,396,053` OOT candidates; the robust and nonrobust -objectives are identical across those three solves. - -![A26 answers the shortlist concern: robust LP value stabilizes for the -reported Prosper and Freddie external applications, and Freddie random pools improve -with candidate cap without overtaking the top-return screen.](../reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.png){#fig-supp-external-candidate-sensitivity width="88%" fig-alt="Line chart of robust LP objective by candidate cap and sampling mode for Prosper and Freddie external replications."} - -![A28 answers the remaining Freddie cap concern: the full `1,396,053`-candidate LP gives the same objective as the large top screens, and the all-candidate optimum funds only loans with rank at most `551`.](../reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.png){#fig-supp-freddie-all-candidate width="88%" fig-alt="Two-panel Freddie FM48 audit showing unchanged robust and nonrobust objectives across 500k, 1M, and all candidates, plus a log-scale comparison of all OOT candidates and worst funded rank."} - -| Dataset | Candidate cap | Candidates solved | Robust LP objective | Nonrobust LP objective | Funded loans | Worst funded rank | -|---|---:|---:|---:|---:|---:|---:| -| Prosper | all | `10,531` | `$199,419` | `$220,260` | `234` | `508` | -| Freddie FM48 | `500,000` | `500,000` | `$1,291,228` | `$1,305,409` | `143` | `551` | -| Freddie FM48 | `1,000,000` | `1,000,000` | `$1,291,228` | `$1,305,409` | `143` | `551` | -| Freddie FM48 | all | `1,396,053` | `$1,291,228` | `$1,305,409` | `143` | `551` | - -: A28. External LP exhaustiveness. The Freddie all-candidate run funds zero -loans outside the top-250,000 screen and therefore converts the former cap -caveat into an auditable dominance certificate. - -Table A27 documents why Freddie/Mendeley FM48 is the reported mortgage -replication. FM24 and FM36 are informative but miss one of the promoted gates; -FM60 keeps high 90% coverage but falls short at alpha = 0.01. FM48 is the only -Freddie horizon that clears the two conformal gates while preserving positive -economic robust value. This is a dataset-level selection audit, not a new search -over the Lending Club frontier. - -A29--A33 record the extended multidataset audit layer. The most important caveat -is Freddie's sparse Mondrian behavior: across all `29` Freddie groups, tiny -groups with only `43` OOT rows drive a minimum reported coverage of `0.5`. -After requiring at least `500` calibration+test rows per group, `25` eligible -groups cover `1,396,010` OOT rows and the minimum 90% coverage rises to -`0.8854`; this is close to, but still below, the nominal 90% threshold. The -paper therefore claims global external coverage and reports group diagnostics, -not perfect conditional validity in every sparse mortgage cell. - -| Audit | Main finding | Caveat | -|---|---|---| -| A29 sparse groups | Eligible Freddie groups cover `1,396,010 / 1,396,053` OOT rows; sparse groups contain only `43` OOT rows. | Minimum eligible 90% coverage is `0.8854`, so this remains diagnostic. | -| A30 intervals | Prosper AUC `0.7073` CI `[0.6956, 0.7190]`; Freddie AUC `0.7839` CI `[0.7799, 0.7878]`. | Robust-objective interval bootstraps funded-loan contributions only. | -| A31 subperiods | Prosper 2012 and 2013 both pass alpha coverage; Freddie quarters keep 90% coverage above target. | Freddie 2015Q4 alpha01 coverage is `0.9896`, just below 99%. | -| A32 Prosper defaults | Main, defaulted-only, and chargedoff-only definitions all pass 90% and alpha01 gates with positive all-candidate robust LP. | Default semantics change default rate and robust objective, so the main status definition remains declared. | -| A33 Freddie segments | Combined FM48 and green pass alpha01; all segment LPs are solved on all candidates with positive robust value. | Red passes 90% coverage but alpha01 is `0.9850`, so it is sensitivity evidence, not a promoted standalone claim. | - -## Cross-Dataset Price of Robustness - -A34 turns the external layer into a positive economic finding rather than a -defensive gate. Using the signed convention -$(\text{nonrobust}-\text{robust})/\text{nonrobust}$, the price of robustness is a -*positive* premium on every frozen external application and -increases with the panel default rate across the cases evaluated. This is a -pattern across two datasets (three Freddie default-window segments plus Prosper), -not a scaling law: four points are consistent with the mechanism but cannot -establish a general relationship. - -| Frozen application | Panel default rate | AUC | Price of robustness | -|---|---:|---:|---:| -| Freddie FM48 (green) | `0.58%` | `0.700` | `+1.00%` | -| Freddie FM48 (combined) | `1.45%` | `0.784` | `+1.09%` | -| Freddie FM48 (red) | `2.97%` | `0.700` | `+2.37%` | -| Prosper final-status | `30.92%` | `0.707` | `+9.46%` | - -: A34. Price of robustness by frozen application, ordered by panel default rate. -The source CSV is -`reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.csv`. - -![A34 answers whether robustness has a stable interpretation outside Lending Club: under frozen external application the premium is positive and increases with the panel default rate.](../reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png){#fig-supp-price-scaling width="82%" fig-alt="Line chart on a log-scale x-axis showing the external price of robustness rising from +1.00 percent to +9.46 percent as the panel default rate increases."} - -Two readings matter. First, the premium tracks irreducible default risk, not -discrimination: the green and red Freddie segments have nearly identical AUC but -different premiums, while their default rates differ by roughly a factor of five. -Higher default risk widens the conformal intervals, so the robust worst case -discounts more economic return. Second, the Lending Club body claim is not read -through this external price table; it is read through the exact A35 -return-bound frontier. The measured headline is therefore narrow: in these -frozen external applications, the conformal robust layer costs at most a -low-double-digit premium, and CRPTO measures which regime a given panel is in. - -## Reviewer Claim Checks - -The table below links the paper's main claims to the evidence surface and -guardrails a reviewer can inspect. The point is not to add another result, but -to make the audit path explicit. - -| Claim | Evidence | Artifact | Test or guardrail | -|---|---|---|---| -| The predictive input is a frozen calibrated PD model, not a refreshed leaderboard model. | AUC, Brier, ECE, temporal backtesting, and calibration diagnostics. | `models/pd_canonical.cbm`, `models/pd_canonical_calibrator.pkl`, paper-facing metric tables. | `EXTRACTION_MANIFEST.json` and champion validation hashes. | -| The conformal layer gives conservative OOT uncertainty on the PD scale. | 90% and 95% coverage, minimum group coverage, and grade/decile audits. | `data/processed/conformal_intervals_mondrian.parquet`. | Conformal validation status and regression tests that check metric consistency across surfaces. | -| The promoted funded set passes the exact safety bound `V <= sqrt(alpha)` (not nominal alpha-coverage). | Body point `V(alpha = 0.01) = 0.035350` (above alpha), `Gamma_CP = 0.162616`, Markov cap `0.345084`, zero violation, `8/8` alpha pass. | A35, A36, exact bound-evaluation parquet, consolidated frontier/governance JSON. | Exact-evaluation file and regression tests. | -| The result is not an isolated lucky policy. | The consolidated frontier has 50,010 deduplicated semantic policies and 27,508 all-alpha above-floor policies; terminal endpoint search has 37,068/37,068 all-alpha passers. | Table A35 and governance files. | Protected search/evaluation split; no continuous-region claim. | -| The supplement strengthens interpretation without moving the body claim. | A20--A34 challenger, dependence, tail-risk, multi-distribution, online, and external-replication diagnostics; A35 is the active frontier; A36--A39 are selected-allocation audits. | Journal robustness tables, Figures 15--25, A35--A39. | Scope caveats in each table; A37--A39 are risk-profile audits, not hidden champion selectors. | -| The frozen PD binary is a faithful paper model. | E3/E4 T1 diagnostics show negligible seed-level discrimination movement and stable expanding-window validation. | `docs/refactor/SENSITIVITY_RUN_DESIGN_2026-06.md` and Appendix E summary. | Non-promoted diagnostics only; they do not replace the champion or become routine reproduction steps. | -| The manuscript is reproducible from frozen evidence. | Tables, figures, Quarto pages, and status reports regenerate from frozen inputs. | Repository code, DVC metadata, rendered book/paper outputs. | Pre-push test and lint hooks, DVC status checks, and manifest validation before release. | - -## Funded-Set Audit Card Status - -The selected body point has an exact aggregate funded-set audit at -`alpha = 0.01`: - -| Quantity | Value | -|---|---:| -| Funded rows at alpha01 | `314` | -| Allocated budget | `$1,000,000` | -| Realized return | `$184,832.48` | -| Weighted true default/loss proxy | `0.035350` | -| Weighted miscoverage `V` | `0.035350` | -| Funded empirical coverage | `0.9427` | -| `Gamma_CP` | `0.162616` | -| Endpoint budget upper | `0.245084` | -| Markov cap | `0.345084` | -| Exact violation | `0.000000` | - -A36 regenerates the row-level funded-set audit from the selected body -allocation and closes the composition card for the live manuscript claim. The -source file is -`reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv`. - -| Grade bucket | Funded rows | Exposure share | Default rate | $V$ contribution | Mean $u_i(0.01)$ | -|---|---:|---:|---:|---:|---:| -| A-B | `2` | `0.51%` | `0.00%` | `0.00000` | `0.11070` | -| C | `85` | `28.71%` | `3.53%` | `0.00495` | `0.14456` | -| D | `174` | `59.43%` | `7.47%` | `0.02690` | `0.26042` | -| E-G | `53` | `11.35%` | `3.77%` | `0.00350` | `0.34866` | - -On a binary default outcome the metric has an exact reading worth stating. A -non-default cannot miss (`y_i = 0 <= u_i`), so `V` counts only funded defaults -whose conformal endpoint does not reach the PD ceiling `u_i = 1`. The conformal -layer's role at the tight `alpha` is to inflate the worst endpoints toward one, -and `V` measures the residual default mass the endpoints did not absorb. This is -why `V` tracks the -funded default rate and exceeds `alpha = 0.01` -- a return-seeking funded set has -a base default rate well above `1%` -- and why no reweighting of the calibration -(group-weighted, localized, or multi-distribution) can drive `V` to the nominal -`alpha` without capping most funded defaults at `u_i = 1`, which would void -`Gamma_CP` and the economics. The honest guarantee is therefore the -`sqrt(alpha)` level, with the deterministic identity holding exactly. - -A37 closes the tail-risk repricing caveat for the selected allocation. -At the baseline `LGD = 0.45`, the selected body point has realized return -`$184,832.48`, weighted default rate `0.035350`, realized CVaR95 loss rate -`0.276211`, and decision-time CVaR95 loss rate `0.218140`. Under the reported -LGD sensitivity grid, repriced return ranges from `$188,367.48` at `LGD = 0.35` -to `$179,529.98` at `LGD = 0.60`. These quantities profile the selected -policy's tail exposure; they do not change the body selector, which is still -the finite-grid return-bound point in A35. - -| LGD | Repriced return | Realized CVaR95 | Decision-time CVaR95 | OCE theta5 realized | Markov cap | -|---:|---:|---:|---:|---:|---:| -| `0.35` | `$188,367.48` | `0.205511` | `0.149497` | `-0.118852` | `0.345084` | -| `0.45` | `$184,832.48` | `0.276211` | `0.218140` | `-0.075978` | `0.345084` | -| `0.60` | `$179,529.98` | `0.382261` | `0.321104` | `0.011384` | `0.345084` | - -: A37. Selected body-point tail-risk repricing from -`reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv`. - -A38 closes the corresponding cluster-bound caveat. At `alpha = 0.01` and -`delta = 0.10`, Markov's body threshold is `0.100000`. The tightest regenerated -cluster-aware Hoeffding threshold is period-grade at `0.281247`; period -(`0.395502`), score-vintage (`0.348546`), and grade bucket (`0.728588`) are also -looser than Markov. The empirical result supports the theorem boundary rather -than weakening it: sharper concentration language would require a less -concentrated funded set or extra assumptions not asserted by the body. - -| Cluster partition | Clusters | Max exposure share | Sum exposure squared | Hoeffding threshold | Tighter than Markov | -|---|---:|---:|---:|---:|:---:| -| period | `11` | `0.185295` | `0.129083` | `0.395502` | `False` | -| grade bucket | `4` | `0.594275` | `0.448512` | `0.728588` | `False` | -| period-grade | `27` | `0.119550` | `0.063907` | `0.281247` | `False` | -| score-vintage | `20` | `0.165295` | `0.099552` | `0.348546` | `False` | - -: A38. Selected body-point cluster-bound audit from -`reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv`. - -A39 closes the final bootstrap diagnostic under the selected allocation. -It resamples funded-loan contributions for the fixed body point (`5,000` draws, -seed `20260702`). The interval is deliberately narrower in scope than a new -statistical guarantee: it does not resample solver inputs, the PD model, -calibration data, conformal intervals, or the finite policy search. - -| Metric | Observed | Bootstrap mean | 2.5% | Median | 97.5% | -|---|---:|---:|---:|---:|---:| -| Return, LGD `0.45` | `$184,832.48` | `$184,623.11` | `$167,963.20` | `$185,098.41` | `$198,650.47` | -| Weighted default / `V` | `0.035350` | `0.035404` | `0.018157` | `0.034601` | `0.057193` | -| `Gamma_CP` | `0.162616` | `0.162317` | `0.137160` | `0.161384` | `0.193092` | -| Realized CVaR95 | `0.276211` | `0.270721` | `0.068928` | `0.266983` | `0.450000` | -| Decision-time CVaR95 | `0.218140` | `0.217210` | `0.205148` | `0.217709` | `0.226318` | - -: A39. Fixed-allocation funded-loan contribution bootstrap from -`reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv`. - -## Why A21--A34 Do Not Strengthen the Main Claim - -A21--A34 are designed to make the paper harder to over-read. A21 shows that a -cluster-aware tightening is transparent but not sharper than Markov under the -observed exposure concentration. A23 shows where weighted, group-weighted, or -multi-distribution conformal methods would matter if CRPTO were recalibrated -under a new protocol. A24 shows that an online controller would have little to -correct on the frozen OOT vintages, but it is still a replay, not evidence from -a live stream. A25--A34 show that the same recipe clears useful gates on Prosper -and Freddie/Mendeley, that Freddie's full candidate universe has been solved, -that subperiod/definition/segment sensitivities are documented, and that the -economic cost of applying the frozen recipe is ordered by panel risk across the -four frozen external applications. They -still do not create a new exact funded-set certificate or theorem for every -external portfolio: they verify candidate-pool exhaustiveness and economic -viability under a frozen recipe, not a new post-selection conformal guarantee. -Together, these evidence files protect the IJDS claim: the submitted result is an -auditable post-hoc conformal robust credit-portfolio decision with an exact -frozen Lending Club funded-set certificate and external economic replication -evidence, not a universal conditional-coverage, cross-dataset, or -online-deployment guarantee. - -## Decision-Certificate Landscape - -The main manuscript positions CRPTO as an auditable post-hoc bridge, not as the -only possible conformal decision framework. The table below clarifies the -certificate landscape that motivated the journal package. - -| Family | Certificate object | What CRPTO uses now | Future-work boundary | -|---|---|---|---| -| Split/Mondrian CP | Marginal or partitioned coverage of PD-scale intervals [@vovk2005; @bostrom2021; @gibbs2024]. | Upper conformal endpoint becomes the robust PD input. | Stronger conditional guarantees require new assumptions or diagnostics. | -| Data-driven robust optimization | Feasibility against an uncertainty set [@bertsimas2004; @goldfarb2003robustportfolio; @delage2010dro; @bertsimas2018datadriven]. | Budgeted robust portfolio with exact funded-set check. | New robust objectives, DRO ambiguity sets, or portfolio-selection variants would be new research lanes. | -| Conformal robustness control | Robustness probability or loss control in downstream decisions [@johnstone2021; @hu2026crc]. | Used as positioning language and audit inspiration. | Not re-promoted as a new CRPTO selector. | -| CROM/CREME/CREDO | Model or decision certificates for robust optimization [@bao2025croms; @zhou2025credo; @zhou2026creme]. | Used to motivate A20--A34 challenger and replication diagnostics. | Could become a CRPTO v2 certificate, but only with a new protocol. | -| Decision-focused learning | Regret-aware training through the optimization loss [@elmachtoub2022; @liu2021riskbounds; @schutte2024robust]. | SPO+ is a comparator in A19. | End-to-end retraining would change the frozen predictive model. | - -## Scientific Upgrade Map - -Several natural extensions are strong paper ideas, but only some can be used in -the current submission without becoming new claims. The distinction below is the -operating rule for revision: use frozen diagnostics to sharpen the boundary, -and reserve promoted claims for separately tagged evidence. - -| Upgrade | Paper improvement available now | Why it is not promoted now | Evidence required for promotion | -|---|---|---|---| -| Tail-aware selector | Use A20--A22 and A37 to show the selected decision's tail profile and the available return-tail trade-off. | The body selector is return-bound, not CVaR/OCE. | Predeclare CVaR/OCE objective or constraint, rerun the finite policy search under a new tag, and exact-audit the selected funded set. | -| Prospective/nested selection | Use A3, A9, A35, and the declared finite-grid denominators to answer post-selection concerns. | The current frontier is a frozen retrospective audit, not a fully prospective clinical-trial-style protocol. | Freeze all selectors before a final untouched evaluation panel or a new dataset, then report the search/evaluation split as the main design. | -| Multi-distribution validity | Use A23 to show where grade, vintage, and fine-cell coverage remain strong or thin. | The intervals were not calibrated by a multi-source or group-weighted objective. | Fit a new conformal layer targeting multi-distribution or group-weighted coverage, then repeat the funded-set audit. | -| Online validity | Use A24 as an OOT vintage replay that documents whether ACI would have needed large corrections. | A replay over historical vintages is not a live sequential guarantee. | Run or simulate a predeclared sequential protocol with online alpha updates and decision-time logging. | -| Decision-focused conformal learner | Use A19 to state the regret-auditability trade-off: SPO+ owns low regret, CRPTO owns auditable funded-set controls. | The PD model is frozen and not trained through the optimizer. | Train an end-to-end learner, calibrate its decision uncertainty, and require the same funded-set certificate as CRPTO. | -| Causal decision layer | Use the discussion to note that observational credit panels support predictive/prescriptive certificates only. | No randomized or quasi-experimental assignment design is present. | Define a causal estimand, identification strategy, and policy-evaluation protocol before promotion. | - -## Coverage-Validity Ladder - -The table below records the validity ladder used to interpret A23--A33. The -purpose is to keep the strongest claims aligned with the available evidence. - -| Level | Claim form | Supporting CRPTO evidence | Boundary | -|---|---|---|---| -| Marginal split CP | Coverage over an exchangeable evaluation population. | OOT interval audit and paper-facing validation tables. | Does not imply profile-level conditional coverage. | -| Mondrian/group CP | Coverage within declared partitions such as score deciles or grades. | Frozen Mondrian intervals and grade diagnostics. | Small grade x vintage cells can remain weak. | -| Weighted / localized coverage | Coverage under known weights or local neighborhoods [@barber2023beyond; @guan2023localized; @jonkers2024wcps]. | A23 reports where reweighting/group focus would matter. | Not fitted as a new interval method. | -| Multi-distribution validity | Coverage across multiple source distributions [@liu2024multisource; @yang2026multidistribution; @bhattacharyya2026groupweighted]. | A23 worst-cell table is a read-only stress test. | Full MDCP would need a new calibration protocol. | -| Online validity | Sequential alpha adaptation [@gibbs2021aci; @liu2026portfolio]. | A24 replays OOT vintages as a static online diagnostic. | Not evidence from a live stream. | -| External economic replication | Frozen recipe transfer to different credit products. | A25--A34 report Prosper and Freddie/Mendeley scoring, conformal, LP, exhaustiveness, sensitivity gates, and price-of-robustness scaling. | Replication evidence, not a new universal guarantee. | - -## Lending-Club And P2P Predecessors - -The table below anchors the empirical domain lineage. These papers narrow the novelty -claim: CRPTO is not the first Lending Club model, not the first P2P portfolio -optimizer, and not the first conformal credit-scoring application. Its claim is -the audited coupling of conformal PD uncertainty with a robust credit-portfolio -decision. - -| Paper family | Domain contribution | CRPTO distinction | -|---|---|---| -| IJDS credit-risk graph learning [@das2023creditgraph]. | Shows that richer financial data structures can improve credit-rating prediction in an IJDS setting. | CRPTO keeps prediction quality as an input and moves the contribution to auditable portfolio decision control. | -| Cost-aware classifier calibration [@yang2025costaware]. | Shows that miscalibration has asymmetric downstream decision costs. | CRPTO uses calibrated PD and conformal upper endpoints as a governance-visible decision input. | -| Lending Club / fintech credit scoring [@jagtiani2019altdata; @albanesi2024credit; @zheng2026twostage]. | Measures predictive and scorecard behavior on platform or fintech lending data. | Uses the PD model as an auditable input to a decision certificate. | -| P2P investment support [@guo2016p2p; @zhao2016p2pportfolio; @babaei2020p2p]. | Combines borrower-level prediction with portfolio-style investment recommendation. | Adds conformal uncertainty and exact alpha-safe funded-set validation. | -| Profit scoring in P2P lending [@serrano2016profitscoring]. | Reframes loan selection around economic return rather than classification accuracy alone. | Adds a post-allocation risk certificate and finite-grid frontier evidence around the economic objective. | -| Robust P2P credit portfolio optimization [@chi2019p2p]. | Brings data-driven robust optimization into P2P lending. | Makes the uncertainty set conformal and traceable. | -| AI/OR digital lending optimization [@aior2025lendingclub]. | Frames Lending Club funding as multi-objective OR. | Keeps risk controls and evidence governance as first-class outputs. | -| Ordinal conformal credit scoring [@kawasumi2026ordinal]. | Applies conformal prediction to credit-score intervals. | CRPTO moves from score uncertainty to a robust portfolio decision. | +The optimizer constrains $\sum_iw_iq_i\le\tau$. It does not constrain $B_u$ +directly, which is why the residual premium remains visible in every audit. -![A reviewer may ask whether another uncertainty baseline dominates; Figure 7 keeps those alternatives as supplement evidence and shows why conformal robust sets remain the promoted input without replacing the champion.](../reports/crpto/figures/crpto_fig7_uncertainty_baselines.png){#fig-supp-uncertainty-baselines width="90%" fig-alt="Three-panel comparison of uncertainty set methods by empirical coverage, mean interval width, and minimum grade coverage."} +## A.3 Deterministic Identity -![A reviewer may ask whether regret training should replace CRPTO; Figure 9 keeps SPO+ as the low-regret comparator, not as a replacement champion.](../reports/crpto/figures/crpto_fig9_spo_regret.png){#fig-supp-spo width="90%" fig-alt="Decision regret comparison of two-stage, SPO+, and conformal robust methods, with SPO+ showing lower regret."} +**Proposition A.1 (funded-set accounting).** For any fixed allocation with +nonnegative normalized weights, outcomes $Y_i\in[0,1]$, upper endpoints +$u_i\in[0,1]$, and +$Z_i=\mathbf1\{Y_i>u_i\}$, -![A19 answers the regret objection by materializing the frontier: SPO+ is the low-regret corner, while CRPTO robust is the auditable-risk-control corner.](../reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png){#fig-supp-regret-auditability width="82%" fig-alt="Scatter plot of regret versus verifiable risk controls, positioning SPO+ as low-regret and CRPTO robust as high-auditability."} - -![CQR is retained as conformal appendix evidence, not as the promoted method.](../reports/crpto/figures/crpto_fig10_cqr_per_grade.png){#fig-supp-cqr width="90%" fig-alt="Coverage by Lending Club grade for CQR and conformal variants, used as appendix evidence rather than promoted method."} +$$ +\sum_iw_iY_i\le B_u+V, +\qquad +B_u=\sum_iw_iu_i, +\quad +V=\sum_iw_iZ_i. +$$ -![A20 answers whether a lower-tail policy was available: the diagnostic surface exposes the return/tail trade-off, while A37 remains the selected-allocation repricing.](../reports/crpto/figures/crpto_fig16_tail_risk_frontier.png){#fig-supp-tail-frontier width="90%" fig-alt="Scatter plot of tail risk versus realized return across a diagnostic policy surface, with lower-tail and higher-return policies marked."} +*Proof.* If $Y_i\le u_i$, then $Y_i\le u_i+Z_i=u_i$. If $Y_i>u_i$, then +$Z_i=1$ and $Y_i\le1\le u_i+1$. Multiply by $w_i$, sum, and use +$w_i\ge0$. $\square$ -![A12 answers how sensitive the frozen funded set is to LGD: CVaR grows with the assumption while OCE and mean loss stay milder, so OCE/CVaR enter as post-hoc stress summaries rather than as the optimized objective.](../reports/crpto/figures/crpto_fig17_tail_risk_lgd.png){#fig-supp-tail-lgd width="82%" fig-alt="Line chart of frozen funded-set loss rate versus LGD (0.35 to 0.60) for CVaR90, CVaR95, CVaR99, OCE and the mean; CVaR grows with LGD while OCE and the mean stay mild."} +The proposition is deterministic and remains true after adaptive allocation. +It is deliberately weak: $V$ is observed only after outcomes become available. -![A22 answers how a tail-constrained challenger would be documented: each point is the highest-return alpha01-safe policy admissible under a decision-time CVaR cap, but the active selector remains the return-bound point.](../reports/crpto/figures/crpto_fig18_tail_constrained_frontier.png){#fig-supp-tail-constrained width="88%" fig-alt="Upward line of realized return versus decision-time CVaR95 cap, with a high-return point and tightest tail cap marked."} +## A.4 Assumption-Conditional Markov Corollary -![A24 answers the online-control objection on the frozen panel: per-vintage and cumulative coverage stay above the 90% target while the Gibbs-Candes ACI target alpha_t barely drifts, so live control remains future protocol.](../reports/crpto/figures/crpto_fig19_online_coverage_aci.png){#fig-supp-online-aci width="88%" fig-alt="Per-vintage and cumulative coverage lines above a 90% target line across eleven OOT quarters, with the ACI target alpha_t on a secondary axis rising slightly from 0.10 to 0.12."} +**Assumption A.1 (weighted funded-set validity).** For the random +calibration/evaluation draw and frozen allocation rule, +$\mathbb E[V]\le\alpha$. -A19--A39 should be read as literature-aligned stress evidence. A19 places CRPTO -against the regret-driven training tradition; A20--A22 translate tail risk and -satisficing into finite-grid and tail-constrained audits without changing the -body claim; A37--A39 regenerate the selected allocation's -tail-risk, concentration, and fixed-allocation bootstrap profile; and A23--A24 show where multi-distribution and online conformal work -would enter if the project moved from a frozen historical panel to a new -protocol. A25--A34 add the -external economic replication layer on Prosper and Freddie/Mendeley, including -Freddie all-candidate exhaustiveness and negative/sparse-cell sensitivities, -while preserving the Lending Club certificate boundary. This keeps the -supplement ambitious without silently changing the submitted method. +This assumption is stronger than marginal or Mondrian coverage. The optimizer +uses $p_i$ and $u_i$ to choose $w_i$, so per-cell conformal validity does not +automatically transfer to funded-dollar weights. -# Appendix D: Fair Lending, MRM, And Governance +**Corollary A.1.** Under Assumption A.1, -The fairness section is a model-risk diagnostic. The public Lending Club data do -not contain direct protected attributes, so the paper cannot claim statutory -fair-lending certification. Where protected attributes are unavailable, the -standard practice is to proxy them--for example via Bayesian Improved Surname -Geocoding [@cfpb2014bisg]--and to interpret machine-learning underwriting -fairness with care [@finreglab2023fairness]. The supplement reports proxy and -intersectional diagnostics to show that the selected funded set does not hide an -obvious weak segment under the available columns. +$$ +\Pr(V\ge t)\le\frac{\alpha}{t},\quad t>0, +$$ -The MRM material documents intended use, out-of-scope use, model assumptions, -calibration and conformal diagnostics, challenger criteria, evidence lineage, -and escalation triggers. In the CRPTO setting, a retraining trigger is not an -automatic production process. It is a research governance event that would -require a new, separately tagged training run and a fresh drift check against the -frozen champion. +and therefore -The governance boundary for the current submission is: +$$ +\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt\alpha\right) +\le\sqrt\alpha. +$$ -| Topic | Current submission | Future work only | +*Proof.* Apply Markov's inequality to nonnegative $V$, set +$t=\sqrt\alpha$, and combine with Proposition A.1. $\square$ + +At active $\alpha=0.10$, the probability bound is `0.316228`. Because this is +loose and assumption-conditional, the paper treats it as sensitivity language, +not as the primary contribution or a deterministic default cap. Formal +selected-set validity would require a dedicated protocol +[@hegazy2025valid_selection_conformal_sets]. + +# Appendix B: Active Evidence A35--A40 + +## A35. Exact Alpha Replay and Saturation + +| Selected | Target $\alpha$ | Used $\alpha$ | Coverage | Avg. width | Min partition coverage | Min grade coverage | $u=1$ rate | +|:---:|---:|---:|---:|---:|---:|---:|---:| +| no | `0.01` | `0.0095` | `0.996720` | `0.988215` | `0.989629` | `0.972270` | `0.935424` | +| no | `0.03` | `0.0285` | `0.988478` | `0.969779` | `0.969799` | `0.966493` | `0.832314` | +| no | `0.05` | `0.0475` | `0.972825` | `0.953580` | `0.950583` | `0.952575` | `0.718156` | +| no | `0.07` | `0.0665` | `0.955271` | `0.795666` | `0.934782` | `0.938443` | `0.613550` | +| yes | `0.10` | `0.0950` | `0.934836` | `0.788879` | `0.926310` | `0.926797` | `0.517873` | +| no | `0.12` | `0.1140` | `0.918265` | `0.783174` | `0.906875` | `0.905738` | `0.454511` | +| no | `0.15` | `0.1425` | `0.886098` | `0.646176` | `0.863562` | `0.863101` | `0.334859` | +| no | `0.20` | `0.1900` | `0.849380` | `0.636585` | `0.829376` | `0.828007` | `0.244668` | + +: A35 exact alpha sensitivity. Source: `crpto_tableA35_exact_alpha_grid.csv`. + +The 99% row is nearly saturated and is not the active policy level. Rows other +than 90% apply the same frozen widening recipe as sensitivity; they are not +separately tuned conformal winners. + +## A36. Calibration Policy Selector + +| Nov. status | Candidate | $\tau$ | $\gamma$ | Nov. objective | Nov. $B_u$ | Dec. $B_u$ | +|---|---|---:|---:|---:|---:|---:| +| **Selected** | `linear-005` | `0.17` | `0.50` | `$99,387.12` | `0.259036` | `0.262082` | +| Eligible | `linear-009` | `0.19` | `0.75` | `$96,446.16` | `0.226723` | `0.228577` | +| Eligible | `linear-002` | `0.15` | `0.50` | `$95,995.22` | `0.226088` | `0.227695` | +| Eligible | `linear-006` | `0.17` | `0.75` | `$93,760.13` | `0.202259` | `0.203504` | +| Eligible | `linear-003` | `0.15` | `0.75` | `$90,893.16` | `0.177614` | `0.178203` | +| Ineligible | `linear-007` | `0.19` | `0.25` | `$113,986.45` | `0.422203` | `0.446378` | +| Ineligible | `linear-004` | `0.17` | `0.25` | `$109,885.56` | `0.375105` | `0.396751` | +| Ineligible | `linear-001` | `0.15` | `0.25` | `$105,398.23` | `0.328413` | `0.343502` | +| Ineligible | `linear-008` | `0.19` | `0.50` | `$102,671.87` | `0.290491` | `0.294861` | + +: A36 complete selector. Source: `crpto_tableA36_calibration_policy_selector.csv`. + +The selector uses `14,943` November rows and the deterministic screen +$B_u\le0.28$. The loader keeps outcomes separate from its 12-column input +frame, which contains no outcome or assumption-conditional quantity. The +selected row remains optimal for caps in +`[0.259036, 0.290491)`. Replaying the same outcome-free rule on `20,695` +December rows selects the same policy. Only then are December outcomes opened: +the midpoint funds 193 loans, returns `$53,313.05`, has weighted default +`0.145650`, weighted miscoverage `0.124925`, and endpoint `0.262082`. The +miscoverage miss is direct evidence that selector stability is not selected-set +coverage validity. + +## A37. Temporal Fixed-Policy Evaluation + +| Period | Policy | Return | Default | Miscoverage | Endpoint | Threshold | +|---|---|---:|---:|---:|---:|---:| +| Full OOT | Selected 50/50 | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | +| Full OOT | Conservative 75% | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | +| Full OOT | Point PD | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | +| 2018H1 | Selected 50/50 | `$92,530.73` | `0.106703` | `0.097428` | `0.259170` | `0.575398` | +| 2018H1 | Point PD | `$118,101.99` | `0.190825` | `0.076425` | `0.938912` | `1.255140` | +| 2018H2 | Selected 50/50 | `$156,185.51` | `0.026725` | `0.026725` | `0.262441` | `0.578669` | +| 2018H2 | Point PD | `$95,603.58` | `0.236728` | `0.113175` | `0.915546` | `1.231774` | +| 2019H1 | Selected 50/50 | `$123,590.69` | `0.077325` | `0.061700` | `0.261872` | `0.578100` | +| 2019H1 | Point PD | `$144,281.46` | `0.170275` | `0.038275` | `0.884838` | `1.201065` | +| 2019H2 | Selected 50/50 | `$110,251.95` | `0.103250` | `0.103250` | `0.261503` | `0.577731` | +| 2019H2 | Point PD | `$256,966.20` | `0.023775` | `0.007900` | `0.898805` | `1.215033` | +| 2020+ | Selected 50/50 | `$99,689.54` | `0.083775` | `0.083775` | `0.264462` | `0.580690` | +| 2020+ | Point PD | `$218,629.14` | `0.016900` | `0.000000` | `0.861425` | `1.177653` | + +: A37 selected temporal rows and matched point-PD rows. Source: +`crpto_tableA37_calibration_selected_temporal_evaluation.csv`. + +Each temporal policy receives a fresh `$1M` budget. The table is intentionally +unfavorable to a universal-dominance reading: point PD is much stronger in +2019H2 and 2020+, whereas CRPTO is much stronger in 2018H2. + +## A38. Funded Letter-Grade Audit + +| Grade | Funded | Exposure share | Default | Miscoverage | Point PD | Midpoint $q$ | Endpoint | Return | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| B | `1` | `0.0040` | `0.000000` | `0.000000` | `0.062075` | `0.101472` | `0.140869` | `$643.20` | +| C | `91` | `0.3136` | `0.028622` | `0.028622` | `0.067801` | `0.108598` | `0.149395` | `$47,303.75` | +| D | `170` | `0.5868` | `0.045839` | `0.045839` | `0.088328` | `0.194926` | `0.301523` | `$109,618.08` | +| E | `33` | `0.0806` | `0.000000` | `0.000000` | `0.085242` | `0.179551` | `0.273860` | `$19,879.71` | +| F | `11` | `0.0130` | `0.269231` | `0.076923` | `0.110970` | `0.425687` | `0.740404` | `$1,266.55` | +| G | `2` | `0.0020` | `0.000000` | `0.000000` | `0.146801` | `0.573401` | `1.000000` | `$616.30` | + +: A38 selected funded-set composition. Source: +`crpto_tableA38_calibration_selected_grade_audit.csv`. + +The table uses letter grade recovered from `sub_grade`; the aligned conformal +group is stored separately. This prevents the score-quantile partition labels +from being misreported as credit grades. + +## A39. Fixed-Allocation Bootstrap + +| Resampling unit | Metric | Observed | 2.5% | 97.5% | +|---|---|---:|---:|---:| +| Month clusters | Realized return | `$179,327.59` | `$163,421.14` | `$193,551.65` | +| Month clusters | Weighted default | `0.039375` | `0.020517` | `0.062622` | +| Month clusters | Weighted miscoverage | `0.036875` | `0.018338` | `0.060084` | +| Month clusters | $\Gamma_{\mathrm{CP}}$ | `0.176102` | `0.132145` | `0.230815` | +| Month clusters | Endpoint budget | `0.258051` | `0.211996` | `0.314870` | +| Funded loans | Realized return | `$179,327.59` | `$162,706.17` | `$193,924.74` | +| Funded loans | Weighted default | `0.039375` | `0.020869` | `0.061742` | +| Funded loans | Weighted miscoverage | `0.036875` | `0.019358` | `0.058812` | +| Funded loans | $\Gamma_{\mathrm{CP}}$ | `0.176102` | `0.137159` | `0.224308` | +| Funded loans | Endpoint budget | `0.258051` | `0.217308` | `0.308571` | + +: A39, 5,000 fixed-allocation resamples per scheme with seed `20260709`. +Month clusters are primary; funded-loan resampling is an independence +sensitivity. Source: `crpto_tableA39_calibration_selected_bootstrap.csv`. The +CSV also records bootstrap means and medians. + +## A40. Matched Decision Audit + +| Policy | Expected objective | Realized return | Default | Miscoverage | Endpoint | Cond. threshold | +|---|---:|---:|---:|---:|---:|---:| +| Selected 50/50 CRPTO | `$168,271.56` | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | +| More-conservative 75% blend | `$160,690.13` | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | +| Point-PD matched-$\tau$ | `$214,019.15` | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | + +: A40 matched full-OOT audit. Source: +`crpto_tableA40_calibration_selected_point_baseline.csv`. + +Selected CRPTO pays `$17,041.55` (`8.678%`) relative to point PD. It reduces +weighted default by `7.9025` percentage points, weighted miscoverage by +`0.5025` percentage points, and the conditional threshold by `66.3266` +percentage points. +Its exact active funded-set quantities are +$\Gamma_{\mathrm{CP}}=0.176102$, +$\Gamma_{\mathrm{res}}=0.088051$, +$B_u=0.258051$, +$B_u+V=0.294926$, and +$B_u+\sqrt{0.10}=0.574279$. + +# Appendix C: Supporting Diagnostics A1--A34 + +A1--A34 were generated before the final simplification. They remain useful for +reviewer questions but are not coequal methods or selectors. + +| Range | Contents | Current role | |---|---|---| -| Fairness | Proxy/intersectional audit on available data. | Direct protected-attribute validation if legally available. | -| Causal automated decisions | Observational credit-risk decisions are reported as predictive/prescriptive certificates only. | Experimental or causal policy evaluation would require a separate design [@fernandezloria2025observational]. | -| Monitoring | Evidence-backed guardrails and MRM triggers. | Live production dashboard. | -| Retraining | No automatic retraining; frozen paper champion. | New named run with drift report. | -| Companion | Quarto + DVC-backed evidence lineage after journal policy allows disclosure. | Streamlit/product showcase. | - -# Appendix E: Reproducibility - -The reviewer-facing reproduction path separates general checks from -evidence-aware validation. This structure mirrors the journal's data/code -expectation: the anonymous submission can describe a reproducible companion -without revealing author identity, and the accepted-paper package can disclose -the public repository, DVC pointers, raw-data instructions, and reproducibility -workflow. The aim is narrower than a replication market or incentive mechanism: -CRPTO uses reproducibility to make one decision certificate auditable, whereas -replication-robust analytics markets study how to allocate value across -strategic data contributors [@falconer2026replication]. - -**PD stability diagnostics.** Two isolated run-tag diagnostics -(`ijds-sensitivity-2026-06-14`, separate from the frozen champion) support -distributing a frozen PD binary rather than a "retrain it yourself" recipe. -Retraining the champion -configuration across three random seeds moves out-of-time AUC within a `0.0006` band, -with Brier stable to `+/- 0.0001` and ECE near `0.006`: the CatBoost -non-reproducibility is negligible at the discrimination level, so the frozen binary is -a faithful representative and the certified evidence bundle is the right reproduction -object. An expanding-window temporal walk-forward keeps internal validation AUC in -`[0.717, 0.733]`, dipping in the most recent window toward the harder post-2018 -out-of-time regime. Both support the stability interpretation rather than overturning -the headline, and are reported as non-promoted T1 diagnostics, not a new champion and -not routine reproduction steps. - -Minimal local checks are: +| A1--A2 | Predictive benchmarks and the historical robustness frontier. | Upstream model and provenance context. | +| A3--A11 | Nested/temporal holdouts, sensitivity, synthetic shift, funded-set loan audit, and exact finalist checks. | Robustness diagnostics; no active hyperparameter selection. | +| A12--A18 | OCE/CVaR, satisficing, dependence, leave-period-out, bootstrap, budget/LGD, and policy-family audits. | Historical mechanism checks. | +| A19 | Synthetic regret-auditability comparison with two-stage and SPO+. | Comparator only; not the `$1M` funded portfolio. | +| A20--A24 | Tail, concentration, multi-distribution, and online-style diagnostics. | Stress evidence; no new conformal guarantee. | +| A25--A34 | Prosper and Freddie/Mendeley external analyses. | Static transfer evidence; not active Lending Club certificates. | -```powershell -just setup-base -just smoke -just paper-submission -``` +The external applications are useful because they cover marketplace personal +loans and single-family mortgages. They use older frozen replication contracts, +not the final nine-cell selector. They therefore support plausibility of the +general PD-to-conformal-to-allocation workflow but cannot be quoted as direct +replications of the active midpoint policy. -Paper-facing output regeneration uses frozen inputs: +Similarly, the synthetic SPO+ experiment answers a different question. It +shows that decision-focused training reduces regret on its own synthetic +optimization task [@elmachtoub2022], whereas the active CRPTO result evaluates +an uncertainty-constrained real-dollar allocation. Combining those values into +one leaderboard would be misleading. -```powershell -just tables -just figures -just evidence -just journal-package -``` +# Appendix D: Reproducibility Protocol + +## D.1 Active Commands -The full release-facing checklist is: +The paper-facing evidence is regenerated from isolated experiment outputs: ```powershell -just lint -just smoke -just validate-champion +just ijds-evidence +uv run pytest tests/test_ijds_active_claim_sync.py -q just paper-submission -uv run pytest tests/test_publication_targets.py -q -uv run dvc status --no-updates +just paper-submission-official +just validate-champion ``` -The strongest check is the validation harness: it recomputes the -promoted Mondrian conformal intervals from the frozen PD binaries and the -recorded recipe (partition edges, calibration-split seed, score scaling, -floor multipliers) and asserts exact agreement with the published interval file -(zero maximum absolute difference per loan and per Mondrian cell under the -locked dependency stack). It is opt-in because it scores the full -calibration and OOT panels: +The stronger methodology replay is explicit and writes only to versioned +experiment paths: ```powershell -$env:CRPTO_RUN_CHAMPION_DRIFT = "1" -uv run pytest tests/test_models/test_conformal_mapie_drift.py -q +just ijds-active-replay ``` -The companion tool `scripts/rebuild_test_predictions_from_frozen.py` -regenerates the canonical test-prediction surface from the same frozen -bundle and refuses to write unless the calibrated scores match the frozen -intervals exactly, which keeps the predictive and decision lineages -permanently tied together. +It is not part of ordinary manuscript rendering because it solves portfolios +and recomputes exact interval grids. The frozen upstream model, calibrator, +historical intervals, and manifest are never overwritten. -Artifact-aware validation additionally requires credentials for the DVC remote: +## D.2 Artifact Lineage -```powershell -uv run dvc status -c -r -``` +| Object | Active source | +|---|---| +| Exact alpha grid | `data/processed/experiments/champion_reopen//conformal/exact_alpha_grid.parquet` | +| Selector grid | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_selection_grid.parquet` | +| OOT evaluation | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_oot_evaluation.csv` | +| Funded allocations | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_full_oot_allocations.parquet` | +| Governance | `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` | +| Paper tables | `reports/crpto/tables/crpto_tableA35...A40_*` | -The following stages must not be rerun as routine reproduction steps: -`crpto.pd.champion`, `crpto.conformal.intervals`, -`crpto.conformal.validation`, `crpto.portfolio.optimization`, and especially -`crpto.portfolio.bound_exact_eval`. Paper-facing commands such as table, -figure, evidence, journal-package, manuscript, supplement, and book renders are -safe because they consume frozen inputs. +The active run is +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. +The exact-alpha run is +`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. -The intended accepted-paper disclosure package has four tiers: +## D.3 Claim-Sync Contract -| Tier | Contents | Reviewer/reader value | -|---|---|---| -| Code and manuscript | Python source, scripts, tests, Quarto body, supplement, and book sources. | Rebuild paper outputs and inspect methodology. | -| Frozen metadata | `EXTRACTION_MANIFEST.json`, DVC metadata, lockfile, status reports, and table/figure provenance. | Verify that claims are tied to immutable files. | -| Data access path | Raw Lending Club source instructions plus external dataset source notes; processed files through the declared DVC remote when allowed. | Reproduce or audit without committing private credentials or raw CSVs to Git. | -| Guardrails | `just smoke`, `just validate-champion`, publication target tests, and DVC status checks. | Separate safe paper reruns from protected champion/search stages. | +`tests/test_ijds_active_claim_sync.py` verifies: -This disclosure plan deliberately avoids local paths, secrets, private tokens, -and author-identifying URLs in the double-anonymous packet. +1. policy mode, $\tau$, $\gamma$, alpha, selector size, and forbidden columns; +2. agreement between governance and A35/A40; +3. presence of every A35--A40 CSV and TeX file; +4. numeric anchors across body, supplement, and official submission TeX; +5. absence of retired headline values from active manuscript surfaces. -# Appendix F: Submission Files +Historical manifest tests remain separate. This separation lets the submitted +claim evolve without rewriting or silently reinterpreting frozen provenance. -The active IJDS submission surfaces are: +# Appendix E: Limitations and Claim Crosswalk -| Surface | Source | Role | +| Claim | Direct evidence | What is not claimed | |---|---|---| -| Anonymous body | `paper/CRPTO_ijds.qmd` | 25-page IJDS-style manuscript source. | -| Online supplement | `paper/supplement_ijds.qmd` | Proofs, A3--A39, MRM/fairness, reproducibility. | -| Long companion | `book/` | Public companion after acceptance or journal-approved disclosure. | -| Publication config | `configs/crpto_publication_targets.yaml` | Venue, template, anonymity, and pivot rules. | - -The active handoff body is mirrored in the official INFORMS IJDS LaTeX template -with double-anonymous settings under `paper/submission/CRPTO_ijds_submission.tex`. -The title page is submitted separately, and repository or remote-storage URLs are -disclosed only according to the journal's data/code and double-anonymous policies. - -# Appendix G: Body Claim Crosswalk - -This crosswalk turns the supplement into a map rather than a long appendix. Each -row links a body-level claim to the appendix evidence that defends it and to the -boundary that prevents overclaiming. - -| Body claim | Body location | Supplement defense | Boundary | -|---|---|---|---| -| CRPTO is a decision method, not a classifier leaderboard. | Abstract, Introduction, Closest Work. | Appendix B A3--A11, Appendix C A19, Figure 1 in the body. | Prediction quality is an input; the certified object is the funded-set decision. | -| The theorem is a Markov bound under weighted funded-set validity. | Theory. | Appendix A proof, Proposition A.1 sharpness, Proposition A.2 cluster sensitivity. | Assumption 1 is a modeling premise, not a universal conformal guarantee. | -| The promoted Lending Club point is exactly auditable. | Results, exact certificate table. | Appendix C A35--A39, funded-set audit card, governance files. | Exact means file-backed accounting on the frozen selected allocation. | -| The selected point is not a lucky singleton. | Finite-grid frontier table. | A35 finite-grid frontier plus terminal and consolidated governance denominators. | Finite declared grid, not continuous global optimality. | -| External credit panels support recipe transfer. | Multi-dataset replication protocol and results. | A25--A34 external replication, exhaustiveness, intervals, subperiod and segment sensitivity. | External evidence is economic replication, not new exact certificates. | -| Regret and auditability are different outputs. | Regret-auditability section. | A19 and Figure 15. | SPO+ owns the synthetic regret corner; CRPTO owns the audited funded-set corner. | -| Tail and concentration evidence interpret the selected point. | Tail risk and distribution robustness. | A20--A24 and A37--A39. | Tail-risk, bootstrap and online/multi-distribution rows are diagnostics, not hidden selectors. | -| Reproducibility is evidence-backed. | Reproducibility and companion. | Appendix E commands, validation harness, DVC/manifest boundary. | Routine reproduction excludes protected champion/search stages. | +| Exact 90% replay | Reference endpoints match to `6.67e-16`; A35. | Independent tuning of every alpha sensitivity row. | +| Outcome-free, assumption-free policy ranking, conditional on the frozen recipe | A36 physically separated outcomes, deterministic endpoint cap, cap-stability interval, and December replay. | A label-free conformal recipe, historically untouched OOT corpus, or preregistration. | +| Independent calibration decision audit | December reselects the midpoint before outcomes are opened; default `0.145650`, miscoverage `0.124925`. | Selected-set conformal validity or a prospective deployment guarantee. | +| Full-OOT return-risk trade-off | A37 and A40. | Causal effect or universal dominance. | +| Funded-set accounting | Proposition A.1 and exact row-level allocation. | Nominal selected-set conformal coverage. | +| Conditional Markov sensitivity | Corollary A.1 under weighted validity. | Deterministic risk cap or sharp tail guarantee. | +| Composition transparency | A38 letter-grade reconciliation. | Legal fair-lending certification. | +| Contribution stability | A39 month-cluster bootstrap with funded-loan sensitivity. | Full pipeline uncertainty or model-selection confidence interval. | +| Reproducibility | Commands, run tags, hashes, and sync tests. | Cross-machine bit-identical model retraining. | + +The active result is intentionally narrow. Optimized OCE/CVaR, online or +non-exchangeable conformal recalibration [@farinhas2024nonexchangeable_crc], +formal valid selection among conformal sets +[@hegazy2025valid_selection_conformal_sets], causal allocation, and live +multi-period deployment would require new protocols and are not hidden +acceptance criteria for this paper. diff --git a/pyproject.toml b/pyproject.toml index 550b6d1..c4a24f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -263,9 +263,22 @@ module = [ "mlflow.*", "duckdb.*", "loguru.*", + "torch", + "torch.*", + "tabpfn", + "tabpfn.*", ] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = [ + "scripts.run_spo_comparison", + "scripts.run_spo_real", + "scripts.run_crpto_vs_spo_stability", + "scripts.experiments.run_tabpfn_tabprep_full", +] +disable_error_code = ["import-not-found"] + # Strict mode for the fully-annotated modules. Legacy code stays under the # global laxer config. Promote modules to this list once they pass cleanly. # Modules in this list have been made fully typed enough for strict checks. diff --git a/reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.pdf b/reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.pdf index 97d6814..370e4a9 100644 Binary files a/reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.pdf and b/reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.pdf differ diff --git a/reports/crpto/figures/crpto_fig14_robust_region_heatmap.pdf b/reports/crpto/figures/crpto_fig14_robust_region_heatmap.pdf index 816f9df..28f5f57 100644 Binary files a/reports/crpto/figures/crpto_fig14_robust_region_heatmap.pdf and b/reports/crpto/figures/crpto_fig14_robust_region_heatmap.pdf differ diff --git a/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf b/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf index 3d541b6..7e7d9de 100644 Binary files a/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf and b/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf differ diff --git a/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf b/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf index bb4b0f0..9162529 100644 Binary files a/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf and b/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf differ diff --git a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf b/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf index 790c211..2c0bf99 100644 Binary files a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf and b/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf differ diff --git a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png b/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png index f36d241..966ab67 100644 Binary files a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png and b/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png differ diff --git a/reports/crpto/tables/README.md b/reports/crpto/tables/README.md index f77c7ae..a0a07fa 100644 --- a/reports/crpto/tables/README.md +++ b/reports/crpto/tables/README.md @@ -2,7 +2,7 @@ This directory holds the CSV/TeX exports consumed by the IJDS paper, the online supplement, and the Quarto book. The paper-facing canon is **`crpto_table0`** -(key metrics) plus the journal appendix package **`A3`–`A39`**. Every one of +(key metrics) plus the journal appendix package **`A3`–`A40`**. Every one of those is cited in the paper, the supplement, or a book chapter (see `book/chapters/07-apendice-robustez.qmd` and `30-replicacion-multidataset.qmd`). @@ -10,16 +10,17 @@ those is cited in the paper, the supplement, or a book chapter (see | Range | Role | | --- | --- | -| `crpto_table0_key_metrics` | Headline metrics block. | +| `crpto_table0_key_metrics` | Headline metrics block, except the retired historical `price_of_robustness*` rows. | | `A3`–`A11` | Holdouts, sensitivity, shift, exact bound eval, funded set. | | `A12`–`A21`, `A21b` | Tail risk, satisficing, dependence, regret-auditability, cluster/concentration bounds. | | `A22`–`A24` | Tail-constrained re-opt, multi-distribution, online ACI diagnostics. | | `A25`–`A34` | External multidataset replication (Prosper/Freddie) and cross-dataset price of robustness. | -| `A35` | Pool93 IJDS finite-grid return-bound frontier and final claim endpoints. | -| `A36` | Pool93 body-point funded-set grade audit regenerated from the selected allocation. | -| `A37` | Pool93 body-point tail-risk repricing regenerated from the selected allocation. | -| `A38` | Pool93 body-point cluster-bound audit regenerated from the selected allocation. | -| `A39` | Pool93 body-point fixed-allocation bootstrap metrics regenerated from the selected allocation. | +| `A35` | Active exact-alpha replay, coverage, width, and endpoint-saturation audit. | +| `A36` | Active November selector, endpoint-cap stability, and December outcome-free replay. | +| `A37` | Active full-OOT and temporal fixed-policy evaluation. | +| `A38` | Active selected funded-set letter-grade composition. | +| `A39` | Active month-cluster bootstrap with funded-loan sensitivity. | +| `A40` | Active selected, conservative, and matched point-PD decision comparison. | ## Legacy / superseded (retained for provenance, not paper-facing) @@ -27,6 +28,14 @@ These early exports are regenerated by upstream stages but are **not** surfaced the paper or book. The A3–A34 journal package is the paper-facing replacement. Do not cite them in new prose; prefer the active set above. +The frozen Lending Club `price_of_robustness=-10.56%` fields in table 0, table +1 and A2 are retained only for provenance. Their historical "nonrobust" +baseline inherited a conformal endpoint constraint and was not a point-PD +comparator. Protected historical pool93 exports retain their explicit +`*_pool93_*` filenames; they are provenance, not the active A35--A40 bundle. +See `docs/research/active_claims_2026-07-04.md` for the active contract and +`docs/research/pool93_certificate_semantics_v2_2026-07-09.md` for history. + - `crpto_table1_robustness_summary` - `crpto_table2_conformal_variant_benchmark` - `crpto_tableA1_benchmark_by_group` diff --git a/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv new file mode 100644 index 0000000..004b318 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv @@ -0,0 +1,9 @@ +selected_for_policy,target_alpha,used_alpha,target_coverage,empirical_coverage,coverage_gap,avg_width,min_partition_coverage,min_grade_coverage,high_endpoint_at_one_rate +False,0.01,0.0095,0.99,0.9967204706919157,0.006720470691915725,0.9882149910656284,0.989628743645835,0.9722703639514731,0.9354243342519386 +False,0.03,0.028499999999999998,0.97,0.9884783056246818,0.01847830562468178,0.9697789837688854,0.9697990273996318,0.9664933564413634,0.8323141991338864 +False,0.05,0.0475,0.95,0.9728246932664907,0.022824693266490725,0.9535799768723978,0.9505830880848586,0.9525750124329999,0.7181555175913519 +False,0.07,0.0665,0.9299999999999999,0.9552712654721186,0.025271265472118665,0.795666199875876,0.9347822665680426,0.938442835829143,0.6135500904760013 +True,0.1,0.095,0.9,0.9348356081757077,0.03483560817570763,0.7888790790793794,0.9263099219620958,0.9267968822615554,0.5178730735474177 +False,0.12,0.11399999999999999,0.88,0.9182645944471934,0.038264594447193434,0.7831737841625059,0.906874767744333,0.9057375416605808,0.4545109781160043 +False,0.15,0.1425,0.85,0.8860977574231856,0.03609775742318566,0.6461762449744832,0.8635623982976592,0.8631010078209027,0.33485872380078663 +False,0.2,0.19,0.8,0.8493800317117481,0.049380031711748096,0.6365853175970958,0.8293756967670011,0.828006532910199,0.24466805601204902 diff --git a/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex new file mode 100644 index 0000000..8af704c --- /dev/null +++ b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex @@ -0,0 +1,14 @@ +\begin{tabular}{rrrrrrrrrr} +\toprule +selected\_for\_policy & target\_alpha & used\_alpha & target\_coverage & empirical\_coverage & coverage\_gap & avg\_width & min\_partition\_coverage & min\_grade\_coverage & high\_endpoint\_at\_one\_rate \\ +\midrule +False & 0.010000 & 0.009500 & 0.990000 & 0.996720 & 0.006720 & 0.988215 & 0.989629 & 0.972270 & 0.935424 \\ +False & 0.030000 & 0.028500 & 0.970000 & 0.988478 & 0.018478 & 0.969779 & 0.969799 & 0.966493 & 0.832314 \\ +False & 0.050000 & 0.047500 & 0.950000 & 0.972825 & 0.022825 & 0.953580 & 0.950583 & 0.952575 & 0.718156 \\ +False & 0.070000 & 0.066500 & 0.930000 & 0.955271 & 0.025271 & 0.795666 & 0.934782 & 0.938443 & 0.613550 \\ +True & 0.100000 & 0.095000 & 0.900000 & 0.934836 & 0.034836 & 0.788879 & 0.926310 & 0.926797 & 0.517873 \\ +False & 0.120000 & 0.114000 & 0.880000 & 0.918265 & 0.038265 & 0.783174 & 0.906875 & 0.905738 & 0.454511 \\ +False & 0.150000 & 0.142500 & 0.850000 & 0.886098 & 0.036098 & 0.646176 & 0.863562 & 0.863101 & 0.334859 \\ +False & 0.200000 & 0.190000 & 0.800000 & 0.849380 & 0.049380 & 0.636585 & 0.829376 & 0.828007 & 0.244668 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv b/reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv index d502833..3518f19 100644 --- a/reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv +++ b/reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv @@ -1,11 +1,10 @@ -role,source_run,candidate_id,policy_family,risk_tolerance,policy_mode,gamma,uncertainty_aversion,realized_return,return_floor_surplus,Gamma_CP_alpha01,V_alpha01,endpoint_budget_upper_alpha01,Markov_cap_alpha01,alpha_grid_pass,n_funded_mean -Minimum Markov-cap endpoint,bound_terminal,10661,claim_bound_terminal_ultra_low_cap,0.16825,capped_blended_uncertainty,0.95,0.6,170467.268819,2.728819,0.095719,0.031875,0.173035950,0.273035950,8/8,311.125 -Low-cap balanced endpoint,bound_terminal,5504,claim_bound_terminal_ultra_low_cap,0.16750,capped_blended_uncertainty,0.925,0.6,171006.195983,541.655983,0.097190,0.031875,0.174789250,0.274789250,8/8,311.125 -Highest return under cap <= 0.30,bound_terminal,36412,claim_bound_terminal_return_recovery,0.17150,blended_uncertainty,0.80,0.5,173314.040806,2849.500806,0.115400,0.035875,0.194580000,0.294580000,8/8,310.875 -Highest return under cap <= 0.32,micro_ext,2021,claim_micro_ext_bound_tight,0.17050,capped_blended_uncertainty,0.645,0.2375,179436.119445,8971.579445,0.139182,0.035875,0.219909610,0.319909610,8/8,310.125 -Highest return under cap <= 0.345,micro_ext,512,claim_micro_ext_body_cap345,0.17225,capped_blended_uncertainty,0.5525,0.0375,184800.413581,14335.873581,0.162562,0.035350,0.244996495,0.344996495,8/8,321.000 -Body/default balanced point,micro_ext,131,claim_micro_ext_body_cap345,0.17150,capped_blended_uncertainty,0.5475,0.05,184832.475845,14367.935845,0.162616,0.035350,0.245083740,0.345083740,8/8,320.500 -Highest return under cap <= 0.36,micro_ext,3212,claim_micro_ext_cap036_return,0.17575,capped_blended_uncertainty,0.525,0.075,186050.727749,15586.187749,0.174600,0.037750,0.258685000,0.358685000,8/8,318.750 -Highest return under cap <= 0.45,expanded,979,bound_efficient_local,0.18500,blended_uncertainty,0.35,0.15,198693.277519,28228.737519,0.252323,0.045600,0.349009950,0.449009950,8/8,310.875 -Highest return under cap <= 0.50,micro,2840,claim_micro_economic_endpoint,0.15825,tail_blended_uncertainty,0.50,0.125,222558.702500,52094.162500,0.459089,0.071075,0.387794500,0.487794500,8/8,238.375 -Max-return economic endpoint,micro_ext,4041,claim_micro_ext_economic_endpoint,0.156875,tail_blended_uncertainty,0.445,0.1375,223458.135875,52993.595875,0.457438,0.069575,0.410753090,0.510753090,8/8,239.625 +role,source_run,candidate_id,policy_family,risk_tolerance,policy_mode,gamma,uncertainty_aversion,realized_return,return_floor_surplus,Gamma_CP_alpha01,Gamma_residual_alpha01,V_alpha01,endpoint_budget_alpha01,endpoint_budget_upper_alpha01,Markov_threshold_alpha01,Markov_cap_alpha01,alpha_grid_pass,n_funded_mean +Minimum Markov-threshold endpoint,bound_terminal,10662,claim_bound_terminal_ultra_low_cap,0.16825,capped_blended_uncertainty,0.95,0.6,170467.268819,2.728819,0.095719,0.004786,0.031875,0.173036,0.173036,0.273036,0.273036,8/8,311.125 +Low-threshold balanced endpoint,bound_terminal,5504,claim_bound_terminal_ultra_low_cap,0.1675,capped_blended_uncertainty,0.925,0.6,171006.195983,541.655983,0.09719,0.007289,0.031875,0.174789,0.174789,0.274789,0.274789,8/8,311.125 +Highest return under threshold <= 0.30,bound_closure,992,claim_bound_closure_low_cap,0.16975,capped_blended_uncertainty,0.75,0.35,174552.51361,4087.97361,0.120988,0.030247,0.035875,0.199997,0.199997,0.299997,0.299997,8/8,306.625 +Highest return under threshold <= 0.32,micro_ext,2021,claim_micro_ext_bound_tight,0.1705,capped_blended_uncertainty,0.645,0.2375,179436.119445,8971.579445,0.139182,0.04941,0.035875,0.21991,0.21991,0.31991,0.31991,8/8,310.125 +Highest return under threshold <= 0.345,micro_ext,512,claim_micro_ext_body_cap345,0.17225,capped_blended_uncertainty,0.5525,0.0375,184800.413581,14335.873581,0.162562,0.072747,0.03535,0.244997,0.244997,0.344997,0.344997,8/8,321.0 +Body/default balanced point,micro_ext,131,claim_micro_ext_body_cap345,0.1715,capped_blended_uncertainty,0.5475,0.05,184832.475845,14367.935845,0.162616,0.073584,0.03535,0.245084,0.245084,0.345084,0.345084,8/8,320.5 +Highest return under threshold <= 0.36,micro_ext,3212,claim_micro_ext_cap036_return,0.17575,capped_blended_uncertainty,0.525,0.075,186050.727749,15586.187749,0.1746,0.082935,0.03775,0.258685,0.258685,0.358685,0.358685,8/8,318.75 +Highest return under threshold <= 0.45,expanded,979,bound_efficient_local,0.185,blended_uncertainty,0.35,0.15,198693.277519,28228.737519,0.252323,0.16401,0.0456,0.34901,0.34901,0.44901,0.44901,8/8,310.875 +Max-return economic endpoint,micro_ext,4041,claim_micro_ext_economic_endpoint,0.156875,tail_blended_uncertainty,0.445,0.1375,223458.135875,52993.595875,0.457438,0.440181,0.069575,0.597056,0.597056,0.697056,0.697056,8/8,239.625 diff --git a/reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex b/reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex index 0de5b64..76f016e 100644 --- a/reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex +++ b/reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex @@ -1,14 +1,15 @@ -\begin{tabular}{llrrrrr} +\begin{tabular}{llrrrrrr} \toprule -Role & Source & Return & $\Gamma_{\mathrm{CP}}$ & $V$ & Markov cap & Pass \\ +Role & Source & Return & $\Gamma_{\mathrm{CP}}$ & $\Gamma_{\mathrm{res}}$ & $V$ & Markov threshold & Pass \\ \midrule -Minimum Markov-cap endpoint & bound terminal & 170,467.27 & 0.095719 & 0.031875 & 0.273036 & 8/8 \\ -Low-cap balanced endpoint & bound terminal & 171,006.20 & 0.097190 & 0.031875 & 0.274789 & 8/8 \\ -Highest return under cap $\leq 0.30$ & bound terminal & 173,314.04 & 0.115400 & 0.035875 & 0.294580 & 8/8 \\ -Highest return under cap $\leq 0.345$ & micro ext & 184,800.41 & 0.162562 & 0.035350 & 0.344996 & 8/8 \\ -Body/default balanced point & micro ext & 184,832.48 & 0.162616 & 0.035350 & 0.345084 & 8/8 \\ -Highest return under cap $\leq 0.36$ & micro ext & 186,050.73 & 0.174600 & 0.037750 & 0.358685 & 8/8 \\ -Highest return under cap $\leq 0.45$ & expanded & 198,693.28 & 0.252323 & 0.045600 & 0.449010 & 8/8 \\ -Max-return economic endpoint & micro ext & 223,458.14 & 0.457438 & 0.069575 & 0.510753 & 8/8 \\ +Minimum Markov-threshold endpoint & bound terminal & 170,467.27 & 0.095719 & 0.004786 & 0.031875 & 0.273036 & 8/8 \\ +Low-threshold balanced endpoint & bound terminal & 171,006.20 & 0.097190 & 0.007289 & 0.031875 & 0.274789 & 8/8 \\ +Highest return under threshold $\leq$ 0.30 & bound closure & 174,552.51 & 0.120988 & 0.030247 & 0.035875 & 0.299997 & 8/8 \\ +Highest return under threshold $\leq$ 0.32 & micro ext & 179,436.12 & 0.139182 & 0.049410 & 0.035875 & 0.319910 & 8/8 \\ +Highest return under threshold $\leq$ 0.345 & micro ext & 184,800.41 & 0.162562 & 0.072747 & 0.035350 & 0.344997 & 8/8 \\ +Body/default balanced point & micro ext & 184,832.48 & 0.162616 & 0.073584 & 0.035350 & 0.345084 & 8/8 \\ +Highest return under threshold $\leq$ 0.36 & micro ext & 186,050.73 & 0.174600 & 0.082935 & 0.037750 & 0.358685 & 8/8 \\ +Highest return under threshold $\leq$ 0.45 & expanded & 198,693.28 & 0.252323 & 0.164010 & 0.045600 & 0.449010 & 8/8 \\ +Max-return economic endpoint & micro ext & 223,458.14 & 0.457438 & 0.440181 & 0.069575 & 0.697056 & 8/8 \\ \bottomrule \end{tabular} diff --git a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv new file mode 100644 index 0000000..1e270cf --- /dev/null +++ b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv @@ -0,0 +1,10 @@ +selected,eligible,audit_selected,audit_eligible,candidate_id,risk_tolerance,gamma,expected_objective,n_funded,weighted_pd_point,weighted_pd_effective,endpoint_budget,audit_expected_objective,audit_n_funded,audit_endpoint_budget +True,True,True,True,linear-005,0.17,0.5,99387.12330098984,162,0.08096395060564901,0.17,0.25903604939435104,103567.0099590662,193,0.2620820323280323 +False,True,False,True,linear-009,0.19,0.75,96446.15634571345,165,0.07983135268116329,0.19,0.22672288243961225,100439.75691860713,192,0.22857742256378005 +False,True,False,True,linear-002,0.15,0.5,95995.21888305439,166,0.07391185908208518,0.15000000000000002,0.2260881409179148,100253.3313725306,190,0.22769467893384143 +False,True,False,True,linear-006,0.17,0.75,93760.12764708955,168,0.0732219020269643,0.17,0.20225936599101194,97897.74942687502,191,0.2035038230017554 +False,True,False,True,linear-003,0.15,0.75,90893.16015692815,167,0.06715742012587558,0.15,0.17761419329137482,95103.61442940257,186,0.17820330775871246 +False,False,False,False,linear-007,0.19,0.25,113986.45107415256,170,0.11259889470623649,0.19,0.42220331588129056,118537.51039714873,194,0.4463778904768255 +False,False,False,False,linear-004,0.17,0.25,109885.56215750676,166,0.10163175926165047,0.16999999999999998,0.3751047222150486,114373.21276113224,196,0.396751177625259 +False,False,False,False,linear-001,0.15,0.25,105398.22903742193,166,0.09052889811149456,0.15,0.3284133056655163,109967.55921988936,194,0.34350244765822646 +False,False,False,False,linear-008,0.19,0.5,102671.8741997059,163,0.08950921111283665,0.19,0.29049078888716334,106734.80147640246,191,0.29486067740758526 diff --git a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex new file mode 100644 index 0000000..1146224 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex @@ -0,0 +1,15 @@ +\begin{tabular}{rrrrlrrrrrrrrrr} +\toprule +selected & eligible & audit\_selected & audit\_eligible & candidate\_id & risk\_tolerance & gamma & expected\_objective & n\_funded & weighted\_pd\_point & weighted\_pd\_effective & endpoint\_budget & audit\_expected\_objective & audit\_n\_funded & audit\_endpoint\_budget \\ +\midrule +True & True & True & True & linear-005 & 0.170000 & 0.500000 & 99387.123301 & 162 & 0.080964 & 0.170000 & 0.259036 & 103567.009959 & 193 & 0.262082 \\ +False & True & False & True & linear-009 & 0.190000 & 0.750000 & 96446.156346 & 165 & 0.079831 & 0.190000 & 0.226723 & 100439.756919 & 192 & 0.228577 \\ +False & True & False & True & linear-002 & 0.150000 & 0.500000 & 95995.218883 & 166 & 0.073912 & 0.150000 & 0.226088 & 100253.331373 & 190 & 0.227695 \\ +False & True & False & True & linear-006 & 0.170000 & 0.750000 & 93760.127647 & 168 & 0.073222 & 0.170000 & 0.202259 & 97897.749427 & 191 & 0.203504 \\ +False & True & False & True & linear-003 & 0.150000 & 0.750000 & 90893.160157 & 167 & 0.067157 & 0.150000 & 0.177614 & 95103.614429 & 186 & 0.178203 \\ +False & False & False & False & linear-007 & 0.190000 & 0.250000 & 113986.451074 & 170 & 0.112599 & 0.190000 & 0.422203 & 118537.510397 & 194 & 0.446378 \\ +False & False & False & False & linear-004 & 0.170000 & 0.250000 & 109885.562158 & 166 & 0.101632 & 0.170000 & 0.375105 & 114373.212761 & 196 & 0.396751 \\ +False & False & False & False & linear-001 & 0.150000 & 0.250000 & 105398.229037 & 166 & 0.090529 & 0.150000 & 0.328413 & 109967.559220 & 194 & 0.343502 \\ +False & False & False & False & linear-008 & 0.190000 & 0.500000 & 102671.874200 & 163 & 0.089509 & 0.190000 & 0.290491 & 106734.801476 & 191 & 0.294861 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv new file mode 100644 index 0000000..9ed9783 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv @@ -0,0 +1,19 @@ +period,policy,risk_tolerance,gamma,n_funded,expected_objective,realized_return,weighted_outcome,weighted_miscoverage,weighted_pd_point,weighted_pd_effective,endpoint_budget,markov_loss_threshold +full_oot,Calibration-selected 50/50 CRPTO,0.17,0.5,308,168271.56287282018,179327.5851322598,0.039375,0.036875,0.0819489105765324,0.17,0.2580510894234676,0.5742788554403055 +full_oot,More-conservative 75% blend,0.17,0.75,312,160690.12704075055,172939.50485426714,0.035875,0.035875,0.0788113673633701,0.1699999999999999,0.2003962108788766,0.5166239768957146 +full_oot,Point-PD matched-tau,0.17,0.0,225,214019.1519816618,196369.14000000004,0.1184,0.0419,0.1538437011518627,0.1538437011518627,0.9213174385285344,1.2375452045453723 +2018H1,Calibration-selected 50/50 CRPTO,0.17,0.5,260,123713.651457411,92530.7342546382,0.106703492661675,0.0974284926616751,0.0808296789072766,0.17,0.2591703210927234,0.5753980871095614 +2018H1,More-conservative 75% blend,0.17,0.75,263,116631.43710632224,75935.74005281724,0.12065,0.11815,0.0758885509922111,0.17,0.2013704830025962,0.5175982490194342 +2018H1,Point-PD matched-tau,0.17,0.0,211,176503.44846593234,118101.99096593227,0.190825,0.076425,0.17,0.17,0.9389119545998964,1.2551397206167343 +2018H2,Calibration-selected 50/50 CRPTO,0.17,0.5,289,138315.09206362677,156185.50718903824,0.026725,0.026725,0.0775592058342477,0.1699999999999998,0.2624407941657518,0.5786685601825898 +2018H2,More-conservative 75% blend,0.17,0.75,286,131160.69145112435,145764.2646632018,0.031025,0.031025,0.0752711349157277,0.1699999999999999,0.2015762883614241,0.517804054378262 +2018H2,Point-PD matched-tau,0.17,0.0,256,192790.6360653248,95603.57631688572,0.2367281267296992,0.113175,0.17,0.17,0.9155458351509776,1.2317736011678155 +2019H1,Calibration-selected 50/50 CRPTO,0.17,0.5,244,137182.83555860247,123590.68683599537,0.077325,0.0617,0.0781281806164286,0.17,0.2618718193835714,0.5780995854004094 +2019H1,More-conservative 75% blend,0.17,0.75,241,129155.4740135068,117840.14770205291,0.0714,0.0608,0.0725032304189913,0.1699999999999999,0.2024989231936695,0.5187266892105076 +2019H1,Point-PD matched-tau,0.17,0.0,203,191975.2452370317,144281.46023703172,0.170275,0.038275,0.17,0.17,0.8848375856531457,1.2010653516699836 +2019H2,Calibration-selected 50/50 CRPTO,0.17,0.5,219,136249.87920464447,110251.95086330731,0.10325,0.10325,0.0784966036859173,0.1699999999999999,0.2615033963140826,0.5777311623309205 +2019H2,More-conservative 75% blend,0.17,0.75,220,127777.26882054852,99327.40175400236,0.10325,0.10325,0.0730478509632306,0.17,0.2023173830122564,0.5185451490290944 +2019H2,Point-PD matched-tau,0.17,0.0,129,197742.1451160193,256966.1951160193,0.023775,0.0079,0.1699999999999999,0.1699999999999999,0.8988048868151989,1.2150326528320368 +2020+,Calibration-selected 50/50 CRPTO,0.17,0.5,166,113753.93195239925,99689.53961659266,0.0837749999999999,0.0837749999999999,0.075537928142652,0.17,0.264462071857348,0.580689837874186 +2020+,More-conservative 75% blend,0.17,0.75,169,106744.29312891632,83162.7196944619,0.093775,0.093775,0.0673508590345457,0.17,0.2042163803218181,0.520444146338656 +2020+,Point-PD matched-tau,0.17,0.0,159,154068.98539948836,218629.1353994884,0.0169,0.0,0.17,0.17,0.861424853142836,1.177652619159674 diff --git a/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex new file mode 100644 index 0000000..29b5993 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex @@ -0,0 +1,24 @@ +\begin{tabular}{llrrrrrrrrrrr} +\toprule +period & policy & risk\_tolerance & gamma & n\_funded & expected\_objective & realized\_return & weighted\_outcome & weighted\_miscoverage & weighted\_pd\_point & weighted\_pd\_effective & endpoint\_budget & markov\_loss\_threshold \\ +\midrule +full\_oot & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 308 & 168271.562873 & 179327.585132 & 0.039375 & 0.036875 & 0.081949 & 0.170000 & 0.258051 & 0.574279 \\ +full\_oot & More-conservative 75\% blend & 0.170000 & 0.750000 & 312 & 160690.127041 & 172939.504854 & 0.035875 & 0.035875 & 0.078811 & 0.170000 & 0.200396 & 0.516624 \\ +full\_oot & Point-PD matched-tau & 0.170000 & 0.000000 & 225 & 214019.151982 & 196369.140000 & 0.118400 & 0.041900 & 0.153844 & 0.153844 & 0.921317 & 1.237545 \\ +2018H1 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 260 & 123713.651457 & 92530.734255 & 0.106703 & 0.097428 & 0.080830 & 0.170000 & 0.259170 & 0.575398 \\ +2018H1 & More-conservative 75\% blend & 0.170000 & 0.750000 & 263 & 116631.437106 & 75935.740053 & 0.120650 & 0.118150 & 0.075889 & 0.170000 & 0.201370 & 0.517598 \\ +2018H1 & Point-PD matched-tau & 0.170000 & 0.000000 & 211 & 176503.448466 & 118101.990966 & 0.190825 & 0.076425 & 0.170000 & 0.170000 & 0.938912 & 1.255140 \\ +2018H2 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 289 & 138315.092064 & 156185.507189 & 0.026725 & 0.026725 & 0.077559 & 0.170000 & 0.262441 & 0.578669 \\ +2018H2 & More-conservative 75\% blend & 0.170000 & 0.750000 & 286 & 131160.691451 & 145764.264663 & 0.031025 & 0.031025 & 0.075271 & 0.170000 & 0.201576 & 0.517804 \\ +2018H2 & Point-PD matched-tau & 0.170000 & 0.000000 & 256 & 192790.636065 & 95603.576317 & 0.236728 & 0.113175 & 0.170000 & 0.170000 & 0.915546 & 1.231774 \\ +2019H1 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 244 & 137182.835559 & 123590.686836 & 0.077325 & 0.061700 & 0.078128 & 0.170000 & 0.261872 & 0.578100 \\ +2019H1 & More-conservative 75\% blend & 0.170000 & 0.750000 & 241 & 129155.474014 & 117840.147702 & 0.071400 & 0.060800 & 0.072503 & 0.170000 & 0.202499 & 0.518727 \\ +2019H1 & Point-PD matched-tau & 0.170000 & 0.000000 & 203 & 191975.245237 & 144281.460237 & 0.170275 & 0.038275 & 0.170000 & 0.170000 & 0.884838 & 1.201065 \\ +2019H2 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 219 & 136249.879205 & 110251.950863 & 0.103250 & 0.103250 & 0.078497 & 0.170000 & 0.261503 & 0.577731 \\ +2019H2 & More-conservative 75\% blend & 0.170000 & 0.750000 & 220 & 127777.268821 & 99327.401754 & 0.103250 & 0.103250 & 0.073048 & 0.170000 & 0.202317 & 0.518545 \\ +2019H2 & Point-PD matched-tau & 0.170000 & 0.000000 & 129 & 197742.145116 & 256966.195116 & 0.023775 & 0.007900 & 0.170000 & 0.170000 & 0.898805 & 1.215033 \\ +2020+ & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 166 & 113753.931952 & 99689.539617 & 0.083775 & 0.083775 & 0.075538 & 0.170000 & 0.264462 & 0.580690 \\ +2020+ & More-conservative 75\% blend & 0.170000 & 0.750000 & 169 & 106744.293129 & 83162.719694 & 0.093775 & 0.093775 & 0.067351 & 0.170000 & 0.204216 & 0.520444 \\ +2020+ & Point-PD matched-tau & 0.170000 & 0.000000 & 159 & 154068.985399 & 218629.135399 & 0.016900 & 0.000000 & 0.170000 & 0.170000 & 0.861425 & 1.177653 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv new file mode 100644 index 0000000..b3a4aa2 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv @@ -0,0 +1,7 @@ +grade,n_funded,exposure,exposure_share,weighted_default_rate,weighted_miscoverage,weighted_pd_point,weighted_pd_effective,weighted_pd_high,realized_return +B,1,4000.0,0.004,0.0,0.0,0.06207486505052294,0.10147180683774551,0.14086874862496807,643.1999999999999 +C,91,313565.9904450649,0.31356599044506495,0.028622364266166715,0.028622364266166715,0.06780118581238306,0.10859786707930257,0.14939454834622204,47303.7528804385 +D,170,586834.0095549352,0.5868340095549351,0.045839197391441946,0.045839197391441946,0.08832773498776167,0.19492559567178316,0.30152345635580474,109618.0772518213 +E,33,80600.0,0.0806,0.0,0.0,0.08524228589130531,0.17955099586752693,0.27385970584374847,19879.705 +F,11,13000.0,0.013000000000000001,0.2692307692307692,0.07692307692307691,0.11097039235072648,0.4256872635040547,0.7404041346573831,1266.55 +G,2,2000.0,0.002,0.0,0.0,0.14680145627400495,0.5734007281370025,1.0,616.3000000000001 diff --git a/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex new file mode 100644 index 0000000..f08cf37 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex @@ -0,0 +1,12 @@ +\begin{tabular}{lrrrrrrrrr} +\toprule +grade & n\_funded & exposure & exposure\_share & weighted\_default\_rate & weighted\_miscoverage & weighted\_pd\_point & weighted\_pd\_effective & weighted\_pd\_high & realized\_return \\ +\midrule +B & 1 & 4000.000000 & 0.004000 & 0.000000 & 0.000000 & 0.062075 & 0.101472 & 0.140869 & 643.200000 \\ +C & 91 & 313565.990445 & 0.313566 & 0.028622 & 0.028622 & 0.067801 & 0.108598 & 0.149395 & 47303.752880 \\ +D & 170 & 586834.009555 & 0.586834 & 0.045839 & 0.045839 & 0.088328 & 0.194926 & 0.301523 & 109618.077252 \\ +E & 33 & 80600.000000 & 0.080600 & 0.000000 & 0.000000 & 0.085242 & 0.179551 & 0.273860 & 19879.705000 \\ +F & 11 & 13000.000000 & 0.013000 & 0.269231 & 0.076923 & 0.110970 & 0.425687 & 0.740404 & 1266.550000 \\ +G & 2 & 2000.000000 & 0.002000 & 0.000000 & 0.000000 & 0.146801 & 0.573401 & 1.000000 & 616.300000 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv new file mode 100644 index 0000000..ebd91b3 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv @@ -0,0 +1,11 @@ +bootstrap_unit,metric,observed,boot_mean,boot_p025,boot_p50,boot_p975,n_units,n_draws,seed,note +origination_month,realized_return,179327.5851322598,179293.01466566528,163421.1394838545,179487.08382041534,193551.6460089551,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +origination_month,weighted_default_rate,0.039375,0.03943546399994348,0.02051693505224659,0.03875339983492982,0.06262154165393485,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +origination_month,weighted_miscoverage,0.036875,0.03691656808682405,0.01833810380778745,0.03613937487855996,0.060083696168230914,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +origination_month,Gamma_CP,0.1761021788469351,0.17637453569205386,0.1321447744854007,0.1746349832245928,0.2308148609432896,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +origination_month,endpoint_budget,0.2580510894234676,0.2583077550716801,0.21199608169774178,0.25666011731200084,0.3148701678446982,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +funded_loan,realized_return,179327.5851322598,179075.4838443055,162706.17200230644,179416.98769601624,193924.73990027257,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +funded_loan,weighted_default_rate,0.039375,0.03963778218343555,0.020869102395698755,0.039043420065914713,0.0617419907402632,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +funded_loan,weighted_miscoverage,0.036875,0.03720263786105997,0.019357859801356386,0.03671487666893709,0.058812406466411934,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +funded_loan,Gamma_CP,0.1761021788469351,0.17602573881357803,0.13715911012280738,0.17480824046170865,0.2243084301758857,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +funded_loan,endpoint_budget,0.2580510894234676,0.25794196914296347,0.21730801774100114,0.2567618086877488,0.30857101650304447,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." diff --git a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex new file mode 100644 index 0000000..dda735b --- /dev/null +++ b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex @@ -0,0 +1,16 @@ +\begin{tabular}{llrrrrrrrrl} +\toprule +bootstrap\_unit & metric & observed & boot\_mean & boot\_p025 & boot\_p50 & boot\_p975 & n\_units & n\_draws & seed & note \\ +\midrule +origination\_month & realized\_return & 179327.585132 & 179293.014666 & 163421.139484 & 179487.083820 & 193551.646009 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +origination\_month & weighted\_default\_rate & 0.039375 & 0.039435 & 0.020517 & 0.038753 & 0.062622 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +origination\_month & weighted\_miscoverage & 0.036875 & 0.036917 & 0.018338 & 0.036139 & 0.060084 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +origination\_month & Gamma\_CP & 0.176102 & 0.176375 & 0.132145 & 0.174635 & 0.230815 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +origination\_month & endpoint\_budget & 0.258051 & 0.258308 & 0.211996 & 0.256660 & 0.314870 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & realized\_return & 179327.585132 & 179075.483844 & 162706.172002 & 179416.987696 & 193924.739900 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & weighted\_default\_rate & 0.039375 & 0.039638 & 0.020869 & 0.039043 & 0.061742 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & weighted\_miscoverage & 0.036875 & 0.037203 & 0.019358 & 0.036715 & 0.058812 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & Gamma\_CP & 0.176102 & 0.176026 & 0.137159 & 0.174808 & 0.224308 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & endpoint\_budget & 0.258051 & 0.257942 & 0.217308 & 0.256762 & 0.308571 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv new file mode 100644 index 0000000..e07feb7 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv @@ -0,0 +1,4 @@ +policy,n_funded,expected_objective,realized_return,weighted_outcome,weighted_miscoverage,endpoint_budget,markov_loss_threshold,return_delta_vs_selected,default_delta_vs_selected,threshold_delta_vs_selected +Calibration-selected 50/50 CRPTO,308,168271.56287282018,179327.5851322598,0.039375,0.036875,0.2580510894234676,0.5742788554403055,0.0,0.0,0.0 +More-conservative 75% blend,312,160690.12704075055,172939.50485426714,0.035875,0.035875,0.2003962108788766,0.5166239768957146,-6388.080277992645,-0.003500000000000003,-0.05765487854459095 +Point-PD matched-tau,225,214019.1519816618,196369.14000000004,0.1184,0.0419,0.9213174385285344,1.2375452045453723,17041.554867740255,0.07902500000000001,0.6632663491050668 diff --git a/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex new file mode 100644 index 0000000..7653717 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex @@ -0,0 +1,9 @@ +\begin{tabular}{lrrrrrrrrrr} +\toprule +policy & n\_funded & expected\_objective & realized\_return & weighted\_outcome & weighted\_miscoverage & endpoint\_budget & markov\_loss\_threshold & return\_delta\_vs\_selected & default\_delta\_vs\_selected & threshold\_delta\_vs\_selected \\ +\midrule +Calibration-selected 50/50 CRPTO & 308 & 168271.562873 & 179327.585132 & 0.039375 & 0.036875 & 0.258051 & 0.574279 & 0.000000 & 0.000000 & 0.000000 \\ +More-conservative 75\% blend & 312 & 160690.127041 & 172939.504854 & 0.035875 & 0.035875 & 0.200396 & 0.516624 & -6388.080278 & -0.003500 & -0.057655 \\ +Point-PD matched-tau & 225 & 214019.151982 & 196369.140000 & 0.118400 & 0.041900 & 0.921317 & 1.237545 & 17041.554868 & 0.079025 & 0.663266 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv b/reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv new file mode 100644 index 0000000..f68ee5f --- /dev/null +++ b/reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv @@ -0,0 +1,3 @@ +policy,realized_return,return_cost_vs_point_pct,n_funded,weighted_default_rate,V_alpha01,Gamma_CP_alpha01,endpoint_budget_alpha01,Markov_threshold_alpha01,expected_return_net_point +Point-PD two-stage LP,196369.14000000004,0.0,225,0.11840000000000002,0.11840000000000002,0.5267355952399213,0.680579296391784,0.780579296391784,214019.1519816618 +Selected CRPTO,184832.47584455396,5.874988379256577,314,0.03535,0.03535,0.1626162784537099,0.24508386600030369,0.3450838660003037,171042.87144858675 diff --git a/reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex b/reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex new file mode 100644 index 0000000..b7edcba --- /dev/null +++ b/reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex @@ -0,0 +1,8 @@ +\begin{tabular}{lrrrrr} +\toprule +Policy & Realized return & Weighted default & $\Gamma_{\mathrm{CP}}$ & $B_u$ & Markov threshold \\ +\midrule +Point-PD two-stage LP & \$196,369.14 & 0.118400 & 0.526736 & 0.680579 & 0.780579 \\ +Selected CRPTO & \$184,832.48 & 0.035350 & 0.162616 & 0.245084 & 0.345084 \\ +\bottomrule +\end{tabular} diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..4ba608f --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,86 @@ +# CRPTO scripts map + +This folder contains both current IJDS publication tooling and historical +research/search entry points. Treat the current submission path as narrow on +purpose. + +## Current IJDS path + +Use these for the active submission workflow: + +- `check_publication_integrity.py` - checks that paper, supplement, README and + official-template docs agree on the active midpoint IJDS claim. +- `build_ijds_calibration_selected_evidence.py` - regenerates active A35--A40 + and governance from the exact-alpha and selected-policy outputs. +- `experiments/run_ijds_exact_alpha_grid_challenger.py` - exact replay into an + isolated experiment path. +- `experiments/run_ijds_calibration_selected_policy_challenger.py` - solves the + declared 3x3 calibration grid and fixed OOT evaluations. +- `compile_ijds_submission.py` - compiles the official INFORMS/IJDS LaTeX + handoff and scans `.log`/`.blg` for unresolved citations or references. +- `run_ty_advisory.py` - runs pinned `ty` in a focused advisory scope for daily + IJDS work or in a blocking full scope. Both scopes are currently clean; + `submission-check` enforces the full scope. +- `build_crpto_journal_package.py` - builds the journal evidence package from + frozen inputs. +- `export_crpto_tables.py` - exports paper tables from frozen artifacts. +- `generate_crpto_figures.py` - exports paper figures from frozen artifacts. +- `render_submission_pdf_previews.py` - creates local HTML-print preview PDFs + for body and supplement. + +The high-level commands are still the source of truth: + +```powershell +just smoke +just type-advisory +just hooks-check +just complexity-report +just ijds-evidence +just paper-submission +just paper-submission-official +just submission-check +``` + +Optional local inspection: + +```powershell +just api-docs-core +``` + +This builds `reports/api-docs/` with `pdoc` for the core optimization, +calibration and evaluation modules. The output is ignored by Git. + +`just complexity-report` runs `radon` over `src/` and `scripts/` and reports +D-or-higher blocks. Treat it as a refactor radar, not a submission gate: some +historical/protected search entry points remain intentionally long until a +post-submission cleanup lane justifies touching them. + +## Protected or historical search paths + +The large scripts under `scripts/search/` and most `scripts/experiments/` are +historical or governed research surfaces. The three active IJDS experiment +modules listed above are the narrow exception. Do not run HPO, conformal interval +generation, champion search, or protected portfolio search unless the work has +a fresh run tag, artifact sink, and drift/revalidation plan. + +`scripts/search/run_conformal_search.py` and +`scripts/search/run_portfolio_search.py` are intentionally retired wrappers. +They now return actionable messages instead of importing the removed generic +`scripts.run_long_pipeline` orchestrator. + +TabPFN, SPO+/PyEPO/Torch and cuOpt remain optional experiment stacks. The +scripts that need them use explicit optional imports so the base IJDS +environment stays light and full-tree type checks still remain useful. + +The active paper should cite A35--A40 and `ijds_policy_governance.json`, not an +ad hoc rerun or the historical policy frontier. + +## Refactor priority + +For pre-submission cleanup, prefer small changes that reduce publication risk: + +1. keep claim synchronization checks strict; +2. keep `mypy` green; +3. use `just type-advisory` for daily IJDS work and `just type-advisory-full` + for final local checks; +4. avoid broad rewrites of protected search code until after IJDS submission. diff --git a/scripts/analyze_crpto_evidence.py b/scripts/analyze_crpto_evidence.py index f1ff1a0..17d65cb 100644 --- a/scripts/analyze_crpto_evidence.py +++ b/scripts/analyze_crpto_evidence.py @@ -14,11 +14,12 @@ import re from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, cast import numpy as np import pandas as pd +from src.optimization.policy import policy_segment_labels from src.utils.script_helpers import load_json, policy_matches, write_json, write_table ROOT = Path(__file__).resolve().parents[1] @@ -426,7 +427,8 @@ def _build_segment_period_table(oot: pd.DataFrame) -> pd.DataFrame: rows: list[dict[str, Any]] = [] total_loan_amount = float(oot["loan_weight"].sum()) grouped = oot.groupby(["period", "original_grade"], dropna=False, observed=True) - for (period, grade), group in grouped: + for key, group in grouped: + period, grade = cast(tuple[object, object], key) row = { "period": str(period), "original_grade": str(grade), @@ -589,23 +591,6 @@ def _interval_arrays_at_alpha( return pd_point, pd_low_90, pd_high_90 -def _segment_labels(frame: pd.DataFrame, policy_mode: str) -> np.ndarray | None: - if str(policy_mode).strip().lower() not in { - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - }: - return None - grade = frame.get("original_grade", pd.Series(["unknown"] * len(frame))).fillna("unknown") - term = frame.get("term", pd.Series(["unknown"] * len(frame))).fillna("unknown") - verification = frame.get( - "verification_status", - pd.Series(["unknown"] * len(frame)), - ).fillna("unknown") - return (grade.astype(str) + "|" + term.astype(str) + "|" + verification.astype(str)).to_numpy( - dtype=object - ) - - def _realized_return( allocation: np.ndarray, loan_amounts: np.ndarray, @@ -626,7 +611,11 @@ def _solve_exact_policy( alpha: float = 0.01, budget: float = 1_000_000.0, ) -> dict[str, Any]: - from src.optimization.portfolio_model import compute_effective_pd, optimize_portfolio_allocation + from src.optimization.portfolio_model import ( + compute_effective_pd, + optimize_portfolio_allocation, + solution_allocation_vector, + ) pd_point, pd_low, pd_high = _interval_arrays_at_alpha(aligned, alpha) effective_pd = compute_effective_pd( @@ -636,7 +625,11 @@ def _solve_exact_policy( gamma=float(policy["gamma"]), delta_cap_quantile=float(policy["delta_cap_quantile"]), tail_focus_quantile=float(policy["tail_focus_quantile"]), - segment_labels=_segment_labels(aligned, str(policy["policy_mode"])), + segment_labels=policy_segment_labels( + aligned, + str(policy["policy_mode"]), + grade_column="original_grade", + ), ) loan_amounts = ( pd.to_numeric(aligned["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) @@ -670,10 +663,7 @@ def _solve_exact_policy( threads=4, solver_backend=str(policy["solver_backend"]), ) - allocation = np.array( - [float(solution["allocation"].get(i, 0.0)) for i in range(len(aligned))], - dtype=float, - ) + allocation = solution_allocation_vector(solution, len(aligned)) total_allocated = float(np.sum(allocation * loan_amounts)) weights = (allocation * loan_amounts) / max(total_allocated, 1e-6) funded_mask = weights > 1e-8 diff --git a/scripts/archive/search/monitor_regret_auditability.py b/scripts/archive/search/monitor_regret_auditability.py index a3cb693..4b8e150 100644 --- a/scripts/archive/search/monitor_regret_auditability.py +++ b/scripts/archive/search/monitor_regret_auditability.py @@ -5,7 +5,6 @@ import argparse import csv import json -import os import re import sqlite3 import subprocess @@ -28,7 +27,7 @@ def emit(text: str = "") -> None: def clear_screen() -> None: """Clear the Windows console.""" - os.system("cls") + subprocess.run(["cmd", "/c", "cls"], check=False) def load_json(path: Path) -> dict[str, Any]: diff --git a/scripts/backtest_conformal_coverage.py b/scripts/backtest_conformal_coverage.py index 7b2f9dd..bcdd899 100644 --- a/scripts/backtest_conformal_coverage.py +++ b/scripts/backtest_conformal_coverage.py @@ -233,18 +233,18 @@ def _build_alerts( if not alerts: return pd.DataFrame( - columns=[ - "level", - "month", - "grade", - "severity", - "n", - "coverage_90", - "coverage_95", - "avg_width_90", - "rule", - "recommended_action", - ] + { + "level": pd.Series(dtype="object"), + "month": pd.Series(dtype="object"), + "grade": pd.Series(dtype="object"), + "severity": pd.Series(dtype="object"), + "n": pd.Series(dtype="int64"), + "coverage_90": pd.Series(dtype="float64"), + "coverage_95": pd.Series(dtype="float64"), + "avg_width_90": pd.Series(dtype="float64"), + "rule": pd.Series(dtype="object"), + "recommended_action": pd.Series(dtype="object"), + } ) return pd.DataFrame(alerts).sort_values(["month", "level", "grade"]).reset_index(drop=True) diff --git a/scripts/benchmark_conformal_variants.py b/scripts/benchmark_conformal_variants.py index 059d384..65741c3 100644 --- a/scripts/benchmark_conformal_variants.py +++ b/scripts/benchmark_conformal_variants.py @@ -9,6 +9,7 @@ import argparse import json import pickle +from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -42,6 +43,48 @@ DEFAULT_POLICY_CONFIG = "configs/crpto_conformal_policy.yaml" +@dataclass(frozen=True) +class BenchmarkData: + model: Any + calibrator: Any | None + cal_df: pd.DataFrame + test_df: pd.DataFrame + features: list[str] + categorical: list[str] + X_cal: pd.DataFrame + y_cal: pd.Series + X_test: pd.DataFrame + y_test: np.ndarray + group_cal: pd.Series + group_test: pd.Series + issue_test: pd.Series + y_prob_cal_raw: np.ndarray + y_prob_test_raw: np.ndarray + y_prob_calibrated: np.ndarray + y_prob_test_calibrated: np.ndarray + prob_cal_lookup: dict[str, np.ndarray] + prob_test_lookup: dict[str, np.ndarray] + + +@dataclass(frozen=True) +class SearchSpace: + partition_candidates: tuple[str, ...] + partition_probability_sources: tuple[str, ...] + n_score_bins_candidates: tuple[int, ...] + fallback_modes: tuple[str, ...] + score_scale_families: tuple[str, ...] + min_group_sizes: tuple[int, ...] + calibration_size_fractions: tuple[float, ...] + + +@dataclass +class VariantResults: + rows: list[dict[str, Any]] = field(default_factory=list) + by_group_rows: list[pd.DataFrame] = field(default_factory=list) + temporal_rows: list[pd.DataFrame] = field(default_factory=list) + local_rows: list[pd.DataFrame] = field(default_factory=list) + + def _summarize_variant( name: str, y_true: np.ndarray, @@ -101,14 +144,14 @@ def _summarize_temporal_stability( "stability_over_time": float("inf"), }, pd.DataFrame( - columns=[ - "variant", - "month", - "n", - "coverage_90", - "avg_width_90", - "coverage_gap_90", - ] + { + "variant": pd.Series(dtype="object"), + "month": pd.Series(dtype="object"), + "n": pd.Series(dtype="int64"), + "coverage_90": pd.Series(dtype="float64"), + "avg_width_90": pd.Series(dtype="float64"), + "coverage_gap_90": pd.Series(dtype="float64"), + } ), ) @@ -210,36 +253,13 @@ def _variant_name( return "::".join(parts) -def main( - alpha: float = 0.10, - selected_config_path: str = "models/conformal_results_mondrian.pkl", - min_group_size_default: int = 500, - cross_cal_sample_size: int = 5000, - cross_test_sample_size: int = 5000, - calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), - partition_candidates: tuple[str, ...] = ( - "grade", - "score_decile_mondrian", - "grade_x_scoreband_mondrian", - ), - partition_probability_sources: tuple[str, ...] = ("raw",), - n_score_bins_candidates: tuple[int, ...] = (10,), - fallback_modes: tuple[str, ...] = ("grade_then_global",), - score_scale_families: tuple[str, ...] = ("none", "bernoulli_sqrt"), - min_group_sizes: tuple[int, ...] | None = None, - artifact_namespace: str | None = None, - calibrator_override_path: str | None = None, - policy_config_path: str = DEFAULT_POLICY_CONFIG, - collect_local_diagnostics: bool = False, -) -> None: - policy = _load_policy_config(policy_config_path).get("policy", {}) or {} +def _load_benchmark_data(calibrator_override_path: str | None) -> BenchmarkData: model, _ = _load_model() calibrator = _load_calibrator(calibrator_override_path) cal_df = read_with_fallback( "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" ) test_df = read_with_fallback("data/processed/test_fe.parquet", "data/processed/test.parquet") - features, categorical = _resolve_features(model, cal_df, test_df) X_cal = _build_feature_matrix(cal_df, features, categorical) y_cal = cal_df[TARGET_COL].astype(float) @@ -260,161 +280,239 @@ def main( if calibrator is not None else np.asarray(y_prob_test_raw, dtype=float) ) - partition_candidates = tuple( - dict.fromkeys(str(x).strip() for x in partition_candidates if str(x).strip()) - ) or ("grade",) - partition_probability_sources = tuple( - dict.fromkeys( - str(x).strip().lower() for x in partition_probability_sources if str(x).strip() - ) - ) or ("raw",) - n_score_bins_candidates = tuple(int(x) for x in n_score_bins_candidates if int(x) > 0) or (10,) - fallback_modes = tuple( - dict.fromkeys(str(x).strip().lower() for x in fallback_modes if str(x).strip()) - ) or ("grade_then_global",) - score_scale_families = tuple( - dict.fromkeys(str(x).strip().lower() for x in score_scale_families if str(x).strip()) - ) or ("none",) - min_group_sizes = tuple( - int(x) for x in (min_group_sizes or (min_group_size_default,)) if int(x) > 0 - ) or (int(min_group_size_default),) - prob_cal_lookup = {"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated} - prob_test_lookup = {"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated} + return BenchmarkData( + model=model, + calibrator=calibrator, + cal_df=cal_df, + test_df=test_df, + features=features, + categorical=categorical, + X_cal=X_cal, + y_cal=y_cal, + X_test=X_test, + y_test=y_test, + group_cal=group_cal, + group_test=group_test, + issue_test=issue_test, + y_prob_cal_raw=y_prob_cal_raw, + y_prob_test_raw=y_prob_test_raw, + y_prob_calibrated=y_prob_calibrated, + y_prob_test_calibrated=y_prob_test_calibrated, + prob_cal_lookup={"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated}, + prob_test_lookup={"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated}, + ) + + +def _unique_clean_strings(values: tuple[str, ...], fallback: tuple[str, ...]) -> tuple[str, ...]: + cleaned = tuple(dict.fromkeys(str(x).strip() for x in values if str(x).strip())) + return cleaned or fallback + + +def _unique_clean_lower_strings( + values: tuple[str, ...], + fallback: tuple[str, ...], +) -> tuple[str, ...]: + cleaned = tuple(dict.fromkeys(str(x).strip().lower() for x in values if str(x).strip())) + return cleaned or fallback + + +def _positive_int_tuple(values: tuple[int, ...], fallback: tuple[int, ...]) -> tuple[int, ...]: + cleaned = tuple(int(x) for x in values if int(x) > 0) + return cleaned or fallback + + +def _min_group_size_tuple( + values: tuple[int, ...] | None, + default: int, +) -> tuple[int, ...]: + return _positive_int_tuple(values or (default,), (int(default),)) + + +def _valid_fraction_tuple(values: tuple[float, ...]) -> tuple[float, ...]: + return tuple(float(x) for x in values if 0 < float(x) <= 1) + + +def _normalize_search_space( + *, + calibration_size_fractions: tuple[float, ...], + partition_candidates: tuple[str, ...], + partition_probability_sources: tuple[str, ...], + n_score_bins_candidates: tuple[int, ...], + fallback_modes: tuple[str, ...], + score_scale_families: tuple[str, ...], + min_group_sizes: tuple[int, ...] | None, + min_group_size_default: int, +) -> SearchSpace: + return SearchSpace( + partition_candidates=_unique_clean_strings(partition_candidates, ("grade",)), + partition_probability_sources=_unique_clean_lower_strings( + partition_probability_sources, ("raw",) + ), + n_score_bins_candidates=_positive_int_tuple(n_score_bins_candidates, (10,)), + fallback_modes=_unique_clean_lower_strings(fallback_modes, ("grade_then_global",)), + score_scale_families=_unique_clean_lower_strings(score_scale_families, ("none",)), + min_group_sizes=_min_group_size_tuple(min_group_sizes, min_group_size_default), + calibration_size_fractions=_valid_fraction_tuple(calibration_size_fractions), + ) + + +def _append_global_variant( + *, + data: BenchmarkData, + results: VariantResults, + alpha: float, +) -> None: + _y_pred_global, y_int_global = create_pd_intervals( + classifier=data.model, + X_cal=data.X_cal, + y_cal=data.y_cal, + X_test=data.X_test, + alpha=alpha, + calibrator=data.calibrator, + ) + row, by_group = _summarize_variant( + "global_split", data.y_test, y_int_global, data.group_test, alpha + ) + temporal_meta, temporal_monthly = _summarize_temporal_stability( + "global_split", data.y_test, y_int_global, data.issue_test + ) + row.update(temporal_meta) + results.rows.append(row) + results.by_group_rows.append(by_group) + results.temporal_rows.append(temporal_monthly) + + +def _append_mondrian_variant( + *, + data: BenchmarkData, + results: VariantResults, partition_cache: dict[ tuple[str, str, int, str, int, float, int], tuple[pd.Series, pd.Series, dict[str, Any]], - ] = {} - - rows: list[dict[str, Any]] = [] - by_group_rows: list[pd.DataFrame] = [] - temporal_rows: list[pd.DataFrame] = [] - local_rows: list[pd.DataFrame] = [] - - def _append_mondrian_variant( - name: str, - *, - partition: str, - partition_probability_source: str, - n_score_bins: int, - fallback_mode: str, - score_scale_family: str, - alpha_used: float = alpha, - min_group_size: int = min_group_size_default, - X_cal_variant: pd.DataFrame | None = None, - y_cal_variant: pd.Series | None = None, - y_prob_cal_variant: np.ndarray | None = None, - base_groups_cal_variant: pd.Series | None = None, - calibration_fraction: float | None = None, - ) -> None: - y_cal_use = y_cal if y_cal_variant is None else y_cal_variant - y_prob_cal_use = ( - prob_cal_lookup[partition_probability_source] - if y_prob_cal_variant is None - else y_prob_cal_variant - ) - y_interval_cal_pred = ( - y_prob_calibrated if y_prob_cal_variant is None else y_prob_cal_variant - ) - base_groups_cal_use = ( - group_cal if base_groups_cal_variant is None else base_groups_cal_variant - ) - cache_key = ( - str(partition), - str(partition_probability_source), - int(n_score_bins), - str(fallback_mode), - int(min_group_size), - float(calibration_fraction or 1.0), - int(len(y_cal_use)), - ) - if cache_key in partition_cache: - group_cal_part, group_test_part, partition_meta = partition_cache[cache_key] - else: - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal_use, - y_prob_eval=prob_test_lookup[partition_probability_source], - partition=partition, - base_groups_cal=base_groups_cal_use.iloc[: len(y_cal_use)].reset_index(drop=True), - base_groups_eval=group_test, - n_score_bins=n_score_bins, - min_group_size=min_group_size, - fallback_mode=fallback_mode, - ) - partition_cache[cache_key] = (group_cal_part, group_test_part, partition_meta) - _y_pred, y_int, _ = create_pd_intervals_mondrian_from_predictions( - y_cal_pred=y_interval_cal_pred, - y_test_pred=y_prob_test_calibrated, - y_cal=y_cal_use, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha_used, + ], + name: str, + partition: str, + partition_probability_source: str, + n_score_bins: int, + fallback_mode: str, + score_scale_family: str, + alpha: float, + alpha_used: float, + min_group_size: int, + collect_local_diagnostics: bool, + y_cal_variant: pd.Series | None = None, + y_prob_cal_variant: np.ndarray | None = None, + base_groups_cal_variant: pd.Series | None = None, + calibration_fraction: float | None = None, +) -> None: + y_cal_use = data.y_cal if y_cal_variant is None else y_cal_variant + y_prob_cal_use = ( + data.prob_cal_lookup[partition_probability_source] + if y_prob_cal_variant is None + else y_prob_cal_variant + ) + y_interval_cal_pred = ( + data.y_prob_calibrated if y_prob_cal_variant is None else y_prob_cal_variant + ) + base_groups_cal_use = ( + data.group_cal if base_groups_cal_variant is None else base_groups_cal_variant + ) + cache_key = ( + str(partition), + str(partition_probability_source), + int(n_score_bins), + str(fallback_mode), + int(min_group_size), + float(calibration_fraction or 1.0), + int(len(y_cal_use)), + ) + if cache_key in partition_cache: + group_cal_part, group_test_part, partition_meta = partition_cache[cache_key] + else: + group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( + y_prob_cal=y_prob_cal_use, + y_prob_eval=data.prob_test_lookup[partition_probability_source], + partition=partition, + base_groups_cal=base_groups_cal_use.iloc[: len(y_cal_use)].reset_index(drop=True), + base_groups_eval=data.group_test, + n_score_bins=n_score_bins, min_group_size=min_group_size, - score_scale_family=score_scale_family, - log_summary=False, - ) - row, by_group = _summarize_variant(name, y_test, y_int, group_test_part, alpha) - temporal_meta, temporal_monthly = _summarize_temporal_stability( - name, y_test, y_int, issue_test + fallback_mode=fallback_mode, ) - row.update(temporal_meta) - row["partition"] = partition_meta.get("partition", partition) - row["partition_probability_source"] = partition_probability_source - row["n_score_bins"] = int(n_score_bins) - row["fallback_mode"] = str(partition_meta.get("fallback_mode", fallback_mode)) - row["scaled_scores"] = bool(score_scale_family != "none") - row["score_scale_family"] = score_scale_family - row["min_group_size"] = int(min_group_size) - row["selected_alpha_used"] = float(alpha_used) - row["fallback_groups_n"] = len(partition_meta.get("fallback_groups", [])) - row["calibration_fraction"] = float(calibration_fraction or 1.0) - rows.append(row) - by_group_rows.append(by_group) - temporal_rows.append(temporal_monthly) - - if collect_local_diagnostics: - local_diag = pd.DataFrame( - { - "record_type": "local_partition_summary", - "variant": name, - "partition": row["partition"], - "group": pd.Series(group_test_part).astype(str), - "y_true": y_test, - "low": y_int[:, 0], - "high": y_int[:, 1], - } - ) - local_diag["covered"] = ( - (local_diag["y_true"] >= local_diag["low"]) - & (local_diag["y_true"] <= local_diag["high"]) - ).astype(float) - local_diag["width"] = local_diag["high"] - local_diag["low"] - local_rows.append(local_diag) - - _y_pred_global, y_int_global = create_pd_intervals( - classifier=model, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=alpha, - calibrator=calibrator, + partition_cache[cache_key] = (group_cal_part, group_test_part, partition_meta) + _y_pred, y_int, _ = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=y_interval_cal_pred, + y_test_pred=data.y_prob_test_calibrated, + y_cal=y_cal_use, + group_cal=group_cal_part, + group_test=group_test_part, + alpha=alpha_used, + min_group_size=min_group_size, + score_scale_family=score_scale_family, + log_summary=False, ) - row, by_group = _summarize_variant("global_split", y_test, y_int_global, group_test, alpha) + row, by_group = _summarize_variant(name, data.y_test, y_int, group_test_part, alpha) temporal_meta, temporal_monthly = _summarize_temporal_stability( - "global_split", y_test, y_int_global, issue_test + name, data.y_test, y_int, data.issue_test ) row.update(temporal_meta) - rows.append(row) - by_group_rows.append(by_group) - temporal_rows.append(temporal_monthly) - - for partition in partition_candidates: - for partition_probability_source in partition_probability_sources: - for n_score_bins in n_score_bins_candidates: - for fallback_mode in fallback_modes: - for score_scale_family in score_scale_families: - for min_group_size in min_group_sizes: + row["partition"] = partition_meta.get("partition", partition) + row["partition_probability_source"] = partition_probability_source + row["n_score_bins"] = int(n_score_bins) + row["fallback_mode"] = str(partition_meta.get("fallback_mode", fallback_mode)) + row["scaled_scores"] = bool(score_scale_family != "none") + row["score_scale_family"] = score_scale_family + row["min_group_size"] = int(min_group_size) + row["selected_alpha_used"] = float(alpha_used) + row["fallback_groups_n"] = len(partition_meta.get("fallback_groups", [])) + row["calibration_fraction"] = float(calibration_fraction or 1.0) + results.rows.append(row) + results.by_group_rows.append(by_group) + results.temporal_rows.append(temporal_monthly) + + if collect_local_diagnostics: + local_diag = pd.DataFrame( + { + "record_type": "local_partition_summary", + "variant": name, + "partition": row["partition"], + "group": pd.Series(group_test_part).astype(str), + "y_true": data.y_test, + "low": y_int[:, 0], + "high": y_int[:, 1], + } + ) + local_diag["covered"] = ( + (local_diag["y_true"] >= local_diag["low"]) + & (local_diag["y_true"] <= local_diag["high"]) + ).astype(float) + local_diag["width"] = local_diag["high"] - local_diag["low"] + results.local_rows.append(local_diag) + + +def _append_search_space_variants( + *, + data: BenchmarkData, + results: VariantResults, + partition_cache: dict[ + tuple[str, str, int, str, int, float, int], + tuple[pd.Series, pd.Series, dict[str, Any]], + ], + space: SearchSpace, + alpha: float, + collect_local_diagnostics: bool, +) -> None: + for partition in space.partition_candidates: + for partition_probability_source in space.partition_probability_sources: + for n_score_bins in space.n_score_bins_candidates: + for fallback_mode in space.fallback_modes: + for score_scale_family in space.score_scale_families: + for min_group_size in space.min_group_sizes: _append_mondrian_variant( - _variant_name( + data=data, + results=results, + partition_cache=partition_cache, + name=_variant_name( partition=partition, partition_probability_source=partition_probability_source, n_score_bins=n_score_bins, @@ -427,40 +525,52 @@ def _append_mondrian_variant( n_score_bins=n_score_bins, fallback_mode=fallback_mode, score_scale_family=score_scale_family, + alpha=alpha, + alpha_used=alpha, min_group_size=min_group_size, + collect_local_diagnostics=collect_local_diagnostics, ) - rng = np.random.RandomState(42) - cal_idx = ( - rng.choice(len(y_cal), size=min(int(cross_cal_sample_size), len(y_cal)), replace=False) - if len(y_cal) > int(cross_cal_sample_size) - else np.arange(len(y_cal)) - ) - test_idx = ( - rng.choice(len(y_test), size=min(int(cross_test_sample_size), len(y_test)), replace=False) - if len(y_test) > int(cross_test_sample_size) - else np.arange(len(y_test)) + +def _sample_indices(n_rows: int, sample_size: int, rng: np.random.RandomState) -> np.ndarray: + return ( + rng.choice(n_rows, size=min(int(sample_size), n_rows), replace=False) + if n_rows > int(sample_size) + else np.arange(n_rows) ) + + +def _append_cross_conformal_variant( + *, + data: BenchmarkData, + results: VariantResults, + alpha: float, + cross_cal_sample_size: int, + cross_test_sample_size: int, +) -> None: + rng = np.random.RandomState(42) + cal_idx = _sample_indices(len(data.y_cal), cross_cal_sample_size, rng) + test_idx = _sample_indices(len(data.y_test), cross_test_sample_size, rng) _y_pred_cross, y_int_cross = create_cross_conformal_score_intervals( - y_cal=y_cal.iloc[cal_idx].reset_index(drop=True), - y_prob_cal=y_prob_calibrated[cal_idx], - y_prob_test=y_prob_test_calibrated[test_idx], + y_cal=data.y_cal.iloc[cal_idx].reset_index(drop=True), + y_prob_cal=data.y_prob_calibrated[cal_idx], + y_prob_test=data.y_prob_test_calibrated[test_idx], alpha=alpha, method="plus", cv=5, ) row, by_group = _summarize_variant( "cross_conformal_score_space", - y_test[test_idx], + data.y_test[test_idx], y_int_cross, - group_test.iloc[test_idx].reset_index(drop=True), + data.group_test.iloc[test_idx].reset_index(drop=True), alpha, ) temporal_meta, temporal_monthly = _summarize_temporal_stability( "cross_conformal_score_space", - y_test[test_idx], + data.y_test[test_idx], y_int_cross, - issue_test.iloc[test_idx].reset_index(drop=True), + data.issue_test.iloc[test_idx].reset_index(drop=True), ) row.update(temporal_meta) row["implementation_note"] = ( @@ -468,108 +578,161 @@ def _append_mondrian_variant( ) row["evaluation_sample_n_cal"] = len(cal_idx) row["evaluation_sample_n_test"] = len(test_idx) - rows.append(row) - by_group_rows.append(by_group) - temporal_rows.append(temporal_monthly) + results.rows.append(row) + results.by_group_rows.append(by_group) + results.temporal_rows.append(temporal_monthly) + +def _append_selected_config_variant( + *, + data: BenchmarkData, + results: VariantResults, + partition_cache: dict[ + tuple[str, str, int, str, int, float, int], + tuple[pd.Series, pd.Series, dict[str, Any]], + ], + selected_config_path: str, + alpha: float, + min_group_size_default: int, + collect_local_diagnostics: bool, +) -> None: cfg_path = Path(selected_config_path) - if cfg_path.exists(): - with open(cfg_path, "rb") as f: - payload = pickle.load(f) - best = payload.get("tuning_90_best", {}) if isinstance(payload, dict) else {} - alpha_used = float(best.get("alpha_used_90", alpha)) - min_group_size = int(best.get("min_group_size", min_group_size_default)) - partition_probability_source = str(best.get("partition_probability_source", "raw")) - n_score_bins = int(best.get("n_score_bins", 10)) - fallback_mode = str(best.get("fallback_mode", "grade_then_global")) - score_scale_family = str(best.get("score_scale_family", "none")) - _append_mondrian_variant( - "mondrian_selected_cfg", - partition=str(best.get("partition", "grade")), - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins, - fallback_mode=fallback_mode, - score_scale_family=score_scale_family, - alpha_used=alpha_used, - min_group_size=min_group_size, - ) + if not cfg_path.exists(): + return + with open(cfg_path, "rb") as f: + payload = pickle.load(f) + best = payload.get("tuning_90_best", {}) if isinstance(payload, dict) else {} + _append_mondrian_variant( + data=data, + results=results, + partition_cache=partition_cache, + name="mondrian_selected_cfg", + partition=str(best.get("partition", "grade")), + partition_probability_source=str(best.get("partition_probability_source", "raw")), + n_score_bins=int(best.get("n_score_bins", 10)), + fallback_mode=str(best.get("fallback_mode", "grade_then_global")), + score_scale_family=str(best.get("score_scale_family", "none")), + alpha=alpha, + alpha_used=float(best.get("alpha_used_90", alpha)), + min_group_size=int(best.get("min_group_size", min_group_size_default)), + collect_local_diagnostics=collect_local_diagnostics, + ) + +def _calibration_sensitivity_rows( + *, + data: BenchmarkData, + space: SearchSpace, + alpha: float, +) -> list[dict[str, Any]]: sensitivity_rows: list[dict[str, Any]] = [] - for frac in calibration_size_fractions: - frac_float = float(frac) - if frac_float <= 0 or frac_float > 1: - continue - cal_df_sub = _subset_calibration_frame(cal_df, calibration_fraction=frac_float) - X_cal_sub = _build_feature_matrix(cal_df_sub, features, categorical) + for frac in space.calibration_size_fractions: + cal_df_sub = _subset_calibration_frame(data.cal_df, calibration_fraction=float(frac)) + X_cal_sub = _build_feature_matrix(cal_df_sub, data.features, data.categorical) y_cal_sub = cal_df_sub[TARGET_COL].astype(float).reset_index(drop=True) group_cal_sub = cal_df_sub[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - y_prob_cal_sub_raw = model.predict_proba(X_cal_sub)[:, 1] + y_prob_cal_sub_raw = data.model.predict_proba(X_cal_sub)[:, 1] y_prob_cal_sub_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_cal_sub_raw) - if calibrator is not None + apply_probability_calibrator(data.calibrator, y_prob_cal_sub_raw) + if data.calibrator is not None else np.asarray(y_prob_cal_sub_raw, dtype=float) ) prob_cal_sub_lookup = {"raw": y_prob_cal_sub_raw, "calibrated": y_prob_cal_sub_calibrated} for partition in ("score_decile_mondrian", "grade_x_scoreband_mondrian"): - for partition_probability_source in partition_probability_sources: - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=prob_cal_sub_lookup[partition_probability_source], - y_prob_eval=prob_test_lookup[partition_probability_source], - partition=partition, - base_groups_cal=group_cal_sub, - base_groups_eval=group_test, - n_score_bins=n_score_bins_candidates[0], - min_group_size=min_group_sizes[0], - fallback_mode=fallback_modes[0], - ) - _y_pred_sub, y_int_sub, _ = create_pd_intervals_mondrian_from_predictions( - y_cal_pred=y_prob_cal_sub_calibrated, - y_test_pred=y_prob_test_calibrated, - y_cal=y_cal_sub, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha, - min_group_size=min_group_sizes[0], - score_scale_family=score_scale_families[0], - log_summary=False, - ) - metrics = validate_coverage(y_test, y_int_sub, alpha=alpha, log_summary=False) - by_group_sub = conditional_coverage_by_group(y_test, y_int_sub, group_test_part) - stability_sub, _ = _summarize_temporal_stability( - partition, y_test, y_int_sub, issue_test - ) + for partition_probability_source in space.partition_probability_sources: sensitivity_rows.append( - { - "record_type": "calibration_size_sensitivity", - "variant": _variant_name( - partition=partition, - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins_candidates[0], - fallback_mode=fallback_modes[0], - score_scale_family=score_scale_families[0], - min_group_size=min_group_sizes[0], - calibration_fraction=frac_float, - ), - "partition": partition_meta.get("partition", partition), - "partition_probability_source": partition_probability_source, - "calibration_fraction": frac_float, - "n_calibration_rows": len(X_cal_sub), - "coverage": float(metrics["empirical_coverage"]), - "coverage_gap": float(metrics["coverage_gap"]), - "avg_width": float(metrics["avg_interval_width"]), - "min_group_coverage": float(by_group_sub["coverage"].min()), - "winkler_90": float( - np.mean( - winkler_interval_score( - y_test, y_int_sub[:, 0], y_int_sub[:, 1], alpha=alpha - ) - ) - ), - "stability_over_time": float(stability_sub["stability_over_time"]), - } + _calibration_sensitivity_row( + data=data, + space=space, + alpha=alpha, + partition=partition, + partition_probability_source=partition_probability_source, + calibration_fraction=float(frac), + X_cal_sub=X_cal_sub, + y_cal_sub=y_cal_sub, + group_cal_sub=group_cal_sub, + y_prob_cal_sub_calibrated=y_prob_cal_sub_calibrated, + y_prob_cal_sub=prob_cal_sub_lookup[partition_probability_source], + ) ) + return sensitivity_rows + - bench = pd.DataFrame(rows) +def _calibration_sensitivity_row( + *, + data: BenchmarkData, + space: SearchSpace, + alpha: float, + partition: str, + partition_probability_source: str, + calibration_fraction: float, + X_cal_sub: pd.DataFrame, + y_cal_sub: pd.Series, + group_cal_sub: pd.Series, + y_prob_cal_sub_calibrated: np.ndarray, + y_prob_cal_sub: np.ndarray, +) -> dict[str, Any]: + group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( + y_prob_cal=y_prob_cal_sub, + y_prob_eval=data.prob_test_lookup[partition_probability_source], + partition=partition, + base_groups_cal=group_cal_sub, + base_groups_eval=data.group_test, + n_score_bins=space.n_score_bins_candidates[0], + min_group_size=space.min_group_sizes[0], + fallback_mode=space.fallback_modes[0], + ) + _y_pred_sub, y_int_sub, _ = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=y_prob_cal_sub_calibrated, + y_test_pred=data.y_prob_test_calibrated, + y_cal=y_cal_sub, + group_cal=group_cal_part, + group_test=group_test_part, + alpha=alpha, + min_group_size=space.min_group_sizes[0], + score_scale_family=space.score_scale_families[0], + log_summary=False, + ) + metrics = validate_coverage(data.y_test, y_int_sub, alpha=alpha, log_summary=False) + by_group_sub = conditional_coverage_by_group(data.y_test, y_int_sub, group_test_part) + stability_sub, _ = _summarize_temporal_stability( + partition, data.y_test, y_int_sub, data.issue_test + ) + return { + "record_type": "calibration_size_sensitivity", + "variant": _variant_name( + partition=partition, + partition_probability_source=partition_probability_source, + n_score_bins=space.n_score_bins_candidates[0], + fallback_mode=space.fallback_modes[0], + score_scale_family=space.score_scale_families[0], + min_group_size=space.min_group_sizes[0], + calibration_fraction=calibration_fraction, + ), + "partition": partition_meta.get("partition", partition), + "partition_probability_source": partition_probability_source, + "calibration_fraction": calibration_fraction, + "n_calibration_rows": len(X_cal_sub), + "coverage": float(metrics["empirical_coverage"]), + "coverage_gap": float(metrics["coverage_gap"]), + "avg_width": float(metrics["avg_interval_width"]), + "min_group_coverage": float(by_group_sub["coverage"].min()), + "winkler_90": float( + np.mean( + winkler_interval_score(data.y_test, y_int_sub[:, 0], y_int_sub[:, 1], alpha=alpha) + ) + ), + "stability_over_time": float(stability_sub["stability_over_time"]), + } + + +def _final_benchmark_frames( + *, + results: VariantResults, + sensitivity_rows: list[dict[str, Any]], + policy: dict[str, Any], +) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]: + bench = pd.DataFrame(results.rows) bench["promotion_pass"] = bench.apply(lambda row: _promotion_pass(row, policy), axis=1) bench = bench.sort_values( [ @@ -584,53 +747,145 @@ def _append_mondrian_variant( ).reset_index(drop=True) bench["selection_rank"] = np.arange(1, len(bench) + 1, dtype=int) bench_by_group = ( - pd.concat(by_group_rows, ignore_index=True) + pd.concat(results.by_group_rows, ignore_index=True) .sort_values(["variant", "group"]) .reset_index(drop=True) ) temporal_diagnostics = ( - pd.concat(temporal_rows, ignore_index=True) + pd.concat(results.temporal_rows, ignore_index=True) .sort_values(["variant", "month"]) .reset_index(drop=True) ) - local_diagnostics = pd.concat(local_rows, ignore_index=True) if local_rows else pd.DataFrame() + local_diagnostics = ( + pd.concat(results.local_rows, ignore_index=True) if results.local_rows else pd.DataFrame() + ) if sensitivity_rows: local_diagnostics = pd.concat( [local_diagnostics, pd.DataFrame(sensitivity_rows)], ignore_index=True, sort=False, ) + return bench, bench_by_group, temporal_diagnostics, local_diagnostics - output_paths = _build_output_paths(artifact_namespace) + +def _selected_config_local_diagnostics( + *, + selected_config_path: str, + selected_intervals_path: Path, +) -> pd.DataFrame: selected_cfg_path = Path(selected_config_path) - selected_intervals_path = output_paths["selected_intervals"] - if selected_cfg_path.exists() and selected_intervals_path.exists(): - with open(selected_cfg_path, "rb") as f: - selected_payload = pickle.load(f) - selected_intervals = pd.read_parquet(selected_intervals_path) - if {"y_true", "pd_low_90", "pd_high_90", GROUP_COL}.issubset(selected_intervals.columns): - selected_local = pd.DataFrame( - { - "record_type": "local_partition_summary", - "variant": "mondrian_selected_cfg", - "partition": str(selected_payload.get("partition", "grade")), - "group": selected_intervals[GROUP_COL].fillna("UNKNOWN").astype(str), - "y_true": pd.to_numeric(selected_intervals["y_true"], errors="coerce"), - "low": pd.to_numeric(selected_intervals["pd_low_90"], errors="coerce"), - "high": pd.to_numeric(selected_intervals["pd_high_90"], errors="coerce"), - } - ) - selected_local["covered"] = ( - (selected_local["y_true"] >= selected_local["low"]) - & (selected_local["y_true"] <= selected_local["high"]) - ).astype(float) - selected_local["width"] = selected_local["high"] - selected_local["low"] - local_diagnostics = pd.concat( - [local_diagnostics, selected_local], - ignore_index=True, - sort=False, - ) + if not selected_cfg_path.exists() or not selected_intervals_path.exists(): + return pd.DataFrame() + with open(selected_cfg_path, "rb") as f: + selected_payload = pickle.load(f) + selected_intervals = pd.read_parquet(selected_intervals_path) + if not {"y_true", "pd_low_90", "pd_high_90", GROUP_COL}.issubset(selected_intervals.columns): + return pd.DataFrame() + selected_local = pd.DataFrame( + { + "record_type": "local_partition_summary", + "variant": "mondrian_selected_cfg", + "partition": str(selected_payload.get("partition", "grade")), + "group": selected_intervals[GROUP_COL].fillna("UNKNOWN").astype(str), + "y_true": pd.to_numeric(selected_intervals["y_true"], errors="coerce"), + "low": pd.to_numeric(selected_intervals["pd_low_90"], errors="coerce"), + "high": pd.to_numeric(selected_intervals["pd_high_90"], errors="coerce"), + } + ) + selected_local["covered"] = ( + (selected_local["y_true"] >= selected_local["low"]) + & (selected_local["y_true"] <= selected_local["high"]) + ).astype(float) + selected_local["width"] = selected_local["high"] - selected_local["low"] + return selected_local + + +def _append_selected_local_diagnostics( + *, + local_diagnostics: pd.DataFrame, + selected_config_path: str, + selected_intervals_path: Path, +) -> pd.DataFrame: + selected_local = _selected_config_local_diagnostics( + selected_config_path=selected_config_path, + selected_intervals_path=selected_intervals_path, + ) + if selected_local.empty: + return local_diagnostics + return pd.concat([local_diagnostics, selected_local], ignore_index=True, sort=False) + + +def _selection_status_payload( + *, + bench: pd.DataFrame, + output_paths: dict[str, Path], + artifact_namespace: str | None, + calibrator_override_path: str | None, + policy_config_path: str, + collect_local_diagnostics: bool, + space: SearchSpace, +) -> dict[str, Any]: + selected = bench.iloc[0].to_dict() + return { + "schema_version": "2026-04-03.1", + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "artifact_namespace": artifact_namespace or "", + "calibrator_override_path": str(calibrator_override_path or ""), + "policy_config_path": str(policy_config_path), + "selected_variant": str(selected.get("variant", "")), + "selection_rank": int(selected.get("selection_rank", 1)), + "promotion_pass": bool(selected.get("promotion_pass", False)), + "selection_criteria": [ + "promotion_pass", + "coverage_gap", + "min_group_coverage", + "winkler_90", + "avg_width", + "stability_over_time", + ], + "retired_backtest_role": ( + "Kupiec/Christoffersen are research diagnostics outside the IJDS " + "promotion gate; validate_conformal_policy.py promotes on material " + "coverage, group coverage, width, alert, and Winkler checks." + ), + "local_diagnostics_mode": ( + "all_variants" if collect_local_diagnostics else "selected_config_plus_sensitivity" + ), + "variants_tested": bench["variant"].astype(str).tolist(), + "report_path": str(output_paths["selection_report"]), + "summary_path": str(output_paths["benchmark"]), + "temporal_diagnostics_path": str(output_paths["temporal_diagnostics"]), + "local_diagnostics_path": str(output_paths["local_diagnostics"]), + "selected_metrics": { + "coverage": float(selected.get("coverage", 0.0)), + "coverage_gap": float(selected.get("coverage_gap", 0.0)), + "avg_width": float(selected.get("avg_width", 0.0)), + "min_group_coverage": float(selected.get("min_group_coverage", 0.0)), + "winkler_90": float(selected.get("winkler_90", 0.0)), + "stability_over_time": float(selected.get("stability_over_time", 0.0)), + }, + "search_space": { + "partition_candidates": list(space.partition_candidates), + "partition_probability_sources": list(space.partition_probability_sources), + "n_score_bins_candidates": [int(x) for x in space.n_score_bins_candidates], + "fallback_modes": list(space.fallback_modes), + "score_scale_families": list(space.score_scale_families), + "min_group_sizes": [int(x) for x in space.min_group_sizes], + "calibration_size_fractions": [float(x) for x in space.calibration_size_fractions], + }, + "top_variants": bench.head(5).to_dict(orient="records"), + } + +def _write_benchmark_outputs( + *, + output_paths: dict[str, Path], + bench: pd.DataFrame, + bench_by_group: pd.DataFrame, + temporal_diagnostics: pd.DataFrame, + local_diagnostics: pd.DataFrame, + status_payload: dict[str, Any], +) -> None: bench_path = output_paths["benchmark"] bench_group_path = output_paths["benchmark_by_group"] selection_path = output_paths["selection_report"] @@ -643,68 +898,9 @@ def _append_mondrian_variant( if not local_diagnostics.empty: local_diagnostics.to_parquet(local_path, index=False) - selected = bench.iloc[0].to_dict() status_path = output_paths["selection_status"] status_path.parent.mkdir(parents=True, exist_ok=True) - status_path.write_text( - json.dumps( - { - "schema_version": "2026-04-03.1", - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "artifact_namespace": artifact_namespace or "", - "calibrator_override_path": str(calibrator_override_path or ""), - "policy_config_path": str(policy_config_path), - "selected_variant": str(selected.get("variant", "")), - "selection_rank": int(selected.get("selection_rank", 1)), - "promotion_pass": bool(selected.get("promotion_pass", False)), - "selection_criteria": [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time", - ], - "retired_backtest_role": ( - "Kupiec/Christoffersen are research diagnostics outside the IJDS " - "promotion gate; validate_conformal_policy.py promotes on material " - "coverage, group coverage, width, alert, and Winkler checks." - ), - "local_diagnostics_mode": ( - "all_variants" - if collect_local_diagnostics - else "selected_config_plus_sensitivity" - ), - "variants_tested": bench["variant"].astype(str).tolist(), - "report_path": str(selection_path), - "summary_path": str(bench_path), - "temporal_diagnostics_path": str(temporal_path), - "local_diagnostics_path": str(local_path), - "selected_metrics": { - "coverage": float(selected.get("coverage", 0.0)), - "coverage_gap": float(selected.get("coverage_gap", 0.0)), - "avg_width": float(selected.get("avg_width", 0.0)), - "min_group_coverage": float(selected.get("min_group_coverage", 0.0)), - "winkler_90": float(selected.get("winkler_90", 0.0)), - "stability_over_time": float(selected.get("stability_over_time", 0.0)), - }, - "search_space": { - "partition_candidates": list(partition_candidates), - "partition_probability_sources": list(partition_probability_sources), - "n_score_bins_candidates": [int(x) for x in n_score_bins_candidates], - "fallback_modes": list(fallback_modes), - "score_scale_families": list(score_scale_families), - "min_group_sizes": [int(x) for x in min_group_sizes], - "calibration_size_fractions": [float(x) for x in calibration_size_fractions], - }, - "top_variants": bench.head(5).to_dict(orient="records"), - }, - indent=2, - default=str, - ), - encoding="utf-8", - ) - + status_path.write_text(json.dumps(status_payload, indent=2, default=str), encoding="utf-8") logger.info("Saved conformal benchmark summary: {} ({})", bench_path, bench.shape) logger.info( "Saved conformal benchmark by-group: {} ({})", bench_group_path, bench_by_group.shape @@ -720,6 +916,103 @@ def _append_mondrian_variant( logger.info("Saved conformal variant selection status: {}", status_path) +def main( + alpha: float = 0.10, + selected_config_path: str = "models/conformal_results_mondrian.pkl", + min_group_size_default: int = 500, + cross_cal_sample_size: int = 5000, + cross_test_sample_size: int = 5000, + calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), + partition_candidates: tuple[str, ...] = ( + "grade", + "score_decile_mondrian", + "grade_x_scoreband_mondrian", + ), + partition_probability_sources: tuple[str, ...] = ("raw",), + n_score_bins_candidates: tuple[int, ...] = (10,), + fallback_modes: tuple[str, ...] = ("grade_then_global",), + score_scale_families: tuple[str, ...] = ("none", "bernoulli_sqrt"), + min_group_sizes: tuple[int, ...] | None = None, + artifact_namespace: str | None = None, + calibrator_override_path: str | None = None, + policy_config_path: str = DEFAULT_POLICY_CONFIG, + collect_local_diagnostics: bool = False, +) -> None: + policy = _load_policy_config(policy_config_path).get("policy", {}) or {} + data = _load_benchmark_data(calibrator_override_path) + space = _normalize_search_space( + calibration_size_fractions=calibration_size_fractions, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + min_group_sizes=min_group_sizes, + min_group_size_default=min_group_size_default, + ) + results = VariantResults() + partition_cache: dict[ + tuple[str, str, int, str, int, float, int], + tuple[pd.Series, pd.Series, dict[str, Any]], + ] = {} + + _append_global_variant(data=data, results=results, alpha=alpha) + _append_search_space_variants( + data=data, + results=results, + partition_cache=partition_cache, + space=space, + alpha=alpha, + collect_local_diagnostics=collect_local_diagnostics, + ) + _append_cross_conformal_variant( + data=data, + results=results, + alpha=alpha, + cross_cal_sample_size=cross_cal_sample_size, + cross_test_sample_size=cross_test_sample_size, + ) + _append_selected_config_variant( + data=data, + results=results, + partition_cache=partition_cache, + selected_config_path=selected_config_path, + alpha=alpha, + min_group_size_default=min_group_size_default, + collect_local_diagnostics=collect_local_diagnostics, + ) + sensitivity_rows = _calibration_sensitivity_rows(data=data, space=space, alpha=alpha) + bench, bench_by_group, temporal_diagnostics, local_diagnostics = _final_benchmark_frames( + results=results, + sensitivity_rows=sensitivity_rows, + policy=policy, + ) + + output_paths = _build_output_paths(artifact_namespace) + local_diagnostics = _append_selected_local_diagnostics( + local_diagnostics=local_diagnostics, + selected_config_path=selected_config_path, + selected_intervals_path=output_paths["selected_intervals"], + ) + status_payload = _selection_status_payload( + bench=bench, + output_paths=output_paths, + artifact_namespace=artifact_namespace, + calibrator_override_path=calibrator_override_path, + policy_config_path=policy_config_path, + collect_local_diagnostics=collect_local_diagnostics, + space=space, + ) + _write_benchmark_outputs( + output_paths=output_paths, + bench=bench, + bench_by_group=bench_by_group, + temporal_diagnostics=temporal_diagnostics, + local_diagnostics=local_diagnostics, + status_payload=status_payload, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--alpha", type=float, default=0.10) diff --git a/scripts/benchmark_pd_set_prediction.py b/scripts/benchmark_pd_set_prediction.py index 1fde082..c987848 100644 --- a/scripts/benchmark_pd_set_prediction.py +++ b/scripts/benchmark_pd_set_prediction.py @@ -5,6 +5,7 @@ import argparse import json import os +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -33,6 +34,46 @@ GROUP_COL = "grade" +@dataclass(frozen=True) +class SetBenchmarkData: + model: Any + calibrator: Any | None + calibrator_name: str + cal_df: pd.DataFrame + test_df: pd.DataFrame + features: list[str] + categorical: list[str] + X_cal: pd.DataFrame + y_cal: pd.Series + X_test: pd.DataFrame + y_test: pd.Series + group_cal: pd.Series + group_test: pd.Series + prob_cal_lookup: dict[str, np.ndarray] + prob_test_lookup: dict[str, np.ndarray] + + +@dataclass(frozen=True) +class SetBenchmarkSettings: + alpha: float + methods: tuple[str, ...] + partitions: tuple[str, ...] + partition_probability_source: str + n_score_bins: int + min_group_size: int + requested_fallback_mode: str + effective_fallback_mode: str + calibration_size_fractions: tuple[float, ...] + + +@dataclass(frozen=True) +class VariantPrediction: + method: str + partition: str + y_pred: np.ndarray + y_sets: np.ndarray + + def _build_output_paths(namespace: str | None = None) -> dict[str, Path]: if namespace: ns = str(namespace).strip().replace("/", "_") @@ -122,6 +163,97 @@ def _normalize_sidecar_fallback_mode(fallback_mode: str) -> str: return "grade_then_global" +def _unique_csv_values(values: tuple[str, ...], fallback: tuple[str, ...]) -> tuple[str, ...]: + cleaned = tuple(dict.fromkeys(str(x).strip() for x in values if str(x).strip())) + return cleaned or fallback + + +def _valid_calibration_fractions(values: tuple[float, ...]) -> tuple[float, ...]: + return tuple(float(x) for x in values if 0 < float(x) <= 1) + + +def _calibrator_name(calibrator: Any | None, calibrator_override_path: str | None) -> str: + if calibrator_override_path: + return Path(str(calibrator_override_path)).stem + if calibrator is not None: + return type(calibrator).__name__ + return "raw" + + +def _load_set_benchmark_data(calibrator_override_path: str | None) -> SetBenchmarkData: + model, _ = _load_model() + calibrator = _load_calibrator(calibrator_override_path) + cal_df = read_with_fallback( + "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" + ) + test_df = read_with_fallback("data/processed/test_fe.parquet", "data/processed/test.parquet") + features, categorical = _resolve_features(model, cal_df, test_df) + X_cal = _build_feature_matrix(cal_df, features, categorical) + y_cal = cal_df[TARGET_COL].astype(int).reset_index(drop=True) + X_test = _build_feature_matrix(test_df, features, categorical) + y_test = test_df[TARGET_COL].astype(int).reset_index(drop=True) + group_cal = cal_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) + group_test = test_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) + y_prob_cal_raw = model.predict_proba(X_cal)[:, 1] + y_prob_test_raw = model.predict_proba(X_test)[:, 1] + y_prob_calibrated = ( + apply_probability_calibrator(calibrator, y_prob_cal_raw) + if calibrator is not None + else np.asarray(y_prob_cal_raw, dtype=float) + ) + y_prob_test_calibrated = ( + apply_probability_calibrator(calibrator, y_prob_test_raw) + if calibrator is not None + else np.asarray(y_prob_test_raw, dtype=float) + ) + return SetBenchmarkData( + model=model, + calibrator=calibrator, + calibrator_name=_calibrator_name(calibrator, calibrator_override_path), + cal_df=cal_df, + test_df=test_df, + features=features, + categorical=categorical, + X_cal=X_cal, + y_cal=y_cal, + X_test=X_test, + y_test=y_test, + group_cal=group_cal, + group_test=group_test, + prob_cal_lookup={"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated}, + prob_test_lookup={"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated}, + ) + + +def _set_benchmark_settings( + *, + alpha: float, + method: str, + methods: tuple[str, ...] | None, + partitions: tuple[str, ...], + partition_probability_source: str, + n_score_bins: int, + min_group_size: int, + fallback_mode: str, + calibration_size_fractions: tuple[float, ...], + prob_cal_lookup: dict[str, np.ndarray], +) -> SetBenchmarkSettings: + source = str(partition_probability_source).strip().lower() or "raw" + if source not in prob_cal_lookup: + raise ValueError(f"Unsupported partition_probability_source: {source}") + return SetBenchmarkSettings( + alpha=float(alpha), + methods=_unique_csv_values(methods or (method,), ("lac",)), + partitions=_unique_csv_values(partitions, ("global",)), + partition_probability_source=source, + n_score_bins=int(n_score_bins), + min_group_size=int(min_group_size), + requested_fallback_mode=str(fallback_mode), + effective_fallback_mode=_normalize_sidecar_fallback_mode(fallback_mode), + calibration_size_fractions=_valid_calibration_fractions(calibration_size_fractions), + ) + + def _make_cases( *, y_true: pd.Series, @@ -164,253 +296,301 @@ def _make_cases( return cases -def main( - alpha: float = 0.10, - method: str = "lac", - methods: tuple[str, ...] | None = None, - partitions: tuple[str, ...] = ("global",), - partition_probability_source: str = "raw", - n_score_bins: int = 10, - min_group_size: int = 500, - fallback_mode: str = "grade_then_global", - calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), - artifact_namespace: str | None = None, - calibrator_override_path: str | None = None, -) -> None: - method_list = tuple(dict.fromkeys(methods or (method,))) or ("lac",) - partition_list = tuple(dict.fromkeys(partitions)) or ("global",) - effective_fallback_mode = _normalize_sidecar_fallback_mode(fallback_mode) +def _is_global_partition(partition: str) -> bool: + return str(partition).strip().lower() == "global" - model, _ = _load_model() - calibrator = _load_calibrator(calibrator_override_path) - calibrator_name = ( - Path(str(calibrator_override_path)).stem - if calibrator_override_path - else type(calibrator).__name__ - if calibrator is not None - else "raw" + +def _predict_variant( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, + method_name: str, + partition_name: str, + X_cal: pd.DataFrame, + y_cal: pd.Series, + group_cal: pd.Series, + y_prob_cal: np.ndarray, +) -> VariantPrediction: + if _is_global_partition(partition_name): + y_pred, y_sets = create_classification_sets( + classifier=data.model, + X_cal=X_cal, + y_cal=y_cal, + X_test=data.X_test, + alpha=settings.alpha, + method=method_name, + calibrator=data.calibrator, + ) + return VariantPrediction( + method=str(method_name), + partition=str(partition_name), + y_pred=y_pred, + y_sets=y_sets, + ) + + group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( + y_prob_cal=y_prob_cal, + y_prob_eval=data.prob_test_lookup[settings.partition_probability_source], + partition=partition_name, + base_groups_cal=group_cal, + base_groups_eval=data.group_test, + n_score_bins=settings.n_score_bins, + min_group_size=settings.min_group_size, + fallback_mode=settings.effective_fallback_mode, ) - cal_df = read_with_fallback( - "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" + y_pred, y_sets, _ = create_classification_sets_mondrian( + classifier=data.model, + X_cal=X_cal, + y_cal=y_cal, + X_test=data.X_test, + group_cal=group_cal_part, + group_test=group_test_part, + alpha=settings.alpha, + method=method_name, + min_group_size=settings.min_group_size, + calibrator=data.calibrator, + ) + return VariantPrediction( + method=str(method_name), + partition=str(partition_meta.get("partition", partition_name)), + y_pred=y_pred, + y_sets=y_sets, ) - test_df = read_with_fallback("data/processed/test_fe.parquet", "data/processed/test.parquet") - features, categorical = _resolve_features(model, cal_df, test_df) - X_cal = _build_feature_matrix(cal_df, features, categorical) - y_cal = cal_df[TARGET_COL].astype(int).reset_index(drop=True) - X_test = _build_feature_matrix(test_df, features, categorical) - y_test = test_df[TARGET_COL].astype(int).reset_index(drop=True) - group_cal = cal_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - group_test = test_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - y_prob_cal_raw = model.predict_proba(X_cal)[:, 1] - y_prob_test_raw = model.predict_proba(X_test)[:, 1] - y_prob_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_cal_raw) - if calibrator is not None - else np.asarray(y_prob_cal_raw, dtype=float) + +def _benchmark_row( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, + prediction: VariantPrediction, +) -> dict[str, Any]: + summary = summarize_prediction_sets( + data.y_test.to_numpy(), prediction.y_pred, prediction.y_sets ) - y_prob_test_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_test_raw) - if calibrator is not None - else np.asarray(y_prob_test_raw, dtype=float) + return { + "method": prediction.method, + "partition": prediction.partition, + "partition_probability_source": settings.partition_probability_source, + "calibrator": data.calibrator_name, + "alpha": settings.alpha, + **{k: float(v) for k, v in summary.items()}, + } + + +def _subsample_calibration_data( + *, + data: SetBenchmarkData, + calibration_fraction: float, +) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, np.ndarray, np.ndarray]: + cal_df_sub = _subset_calibration_frame(data.cal_df, calibration_fraction=calibration_fraction) + X_cal_sub = _build_feature_matrix(cal_df_sub, data.features, data.categorical) + y_cal_sub = cal_df_sub[TARGET_COL].astype(int).reset_index(drop=True) + group_cal_sub = cal_df_sub[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) + y_prob_cal_sub_raw = data.model.predict_proba(X_cal_sub)[:, 1] + y_prob_cal_sub_calibrated = ( + apply_probability_calibrator(data.calibrator, y_prob_cal_sub_raw) + if data.calibrator is not None + else np.asarray(y_prob_cal_sub_raw, dtype=float) + ) + return ( + cal_df_sub, + X_cal_sub, + y_cal_sub, + group_cal_sub, + y_prob_cal_sub_raw, + y_prob_cal_sub_calibrated, + ) + + +def _sensitivity_row( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, + method_name: str, + partition_name: str, + calibration_fraction: float, +) -> dict[str, Any]: + _cal_df_sub, X_cal_sub, y_cal_sub, group_cal_sub, y_prob_raw, y_prob_calibrated = ( + _subsample_calibration_data(data=data, calibration_fraction=calibration_fraction) + ) + y_prob_cal = y_prob_raw if settings.partition_probability_source == "raw" else y_prob_calibrated + prediction = _predict_variant( + data=data, + settings=settings, + method_name=method_name, + partition_name=partition_name, + X_cal=X_cal_sub, + y_cal=y_cal_sub, + group_cal=group_cal_sub, + y_prob_cal=y_prob_cal, + ) + summary_sub = summarize_prediction_sets( + data.y_test.to_numpy(), prediction.y_pred, prediction.y_sets ) - prob_cal_lookup = {"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated} - prob_test_lookup = {"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated} - partition_probability_source = str(partition_probability_source).strip().lower() or "raw" - if partition_probability_source not in prob_cal_lookup: - raise ValueError( - f"Unsupported partition_probability_source: {partition_probability_source}" + return { + "method": str(method_name), + "partition": str(prediction.partition), + "partition_probability_source": settings.partition_probability_source, + "calibrator": data.calibrator_name, + "calibration_fraction": float(calibration_fraction), + "n_calibration_rows": int(len(X_cal_sub)), + "set_coverage": float(summary_sub["set_coverage"]), + "singleton_rate": float(summary_sub["singleton_rate"]), + "ambiguity_rate": float(summary_sub["ambiguity_rate"]), + "empty_set_rate": float(summary_sub["empty_set_rate"]), + "default_rate_ambiguous": float(summary_sub["default_rate_ambiguous"]), + } + + +def _sensitivity_rows_for_variant( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, + method_name: str, + partition_name: str, +) -> list[dict[str, Any]]: + return [ + _sensitivity_row( + data=data, + settings=settings, + method_name=method_name, + partition_name=partition_name, + calibration_fraction=float(frac), ) + for frac in settings.calibration_size_fractions + ] + +def _run_benchmark_matrix( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, +) -> tuple[ + pd.DataFrame, + dict[tuple[str, str], pd.DataFrame], + dict[tuple[str, str], list[dict[str, Any]]], +]: benchmark_rows: list[dict[str, Any]] = [] cases_by_variant: dict[tuple[str, str], pd.DataFrame] = {} sensitivity_by_variant: dict[tuple[str, str], list[dict[str, Any]]] = {} - for method_name in method_list: - for partition_name in partition_list: - if str(partition_name).strip().lower() == "global": - y_pred, y_sets = create_classification_sets( - classifier=model, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=alpha, - method=method_name, - calibrator=calibrator, - ) - else: - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=prob_cal_lookup[partition_probability_source], - y_prob_eval=prob_test_lookup[partition_probability_source], - partition=partition_name, - base_groups_cal=group_cal, - base_groups_eval=group_test, - n_score_bins=n_score_bins, - min_group_size=min_group_size, - fallback_mode=effective_fallback_mode, - ) - y_pred, y_sets, _ = create_classification_sets_mondrian( - classifier=model, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha, - method=method_name, - min_group_size=min_group_size, - calibrator=calibrator, - ) - partition_name = str(partition_meta.get("partition", partition_name)) - - summary = summarize_prediction_sets(y_test.to_numpy(), y_pred, y_sets) + for method_name in settings.methods: + for partition_name in settings.partitions: + prediction = _predict_variant( + data=data, + settings=settings, + method_name=method_name, + partition_name=partition_name, + X_cal=data.X_cal, + y_cal=data.y_cal, + group_cal=data.group_cal, + y_prob_cal=data.prob_cal_lookup[settings.partition_probability_source], + ) benchmark_rows.append( - { - "method": str(method_name), - "partition": str(partition_name), - "partition_probability_source": str(partition_probability_source), - "calibrator": str(calibrator_name), - "alpha": float(alpha), - **{k: float(v) for k, v in summary.items()}, - } + _benchmark_row(data=data, settings=settings, prediction=prediction) ) - cases_by_variant[(str(method_name), str(partition_name))] = _make_cases( - y_true=y_test, - y_pred=y_pred, - y_sets=y_sets, - test_df=test_df, - method=str(method_name), - partition=str(partition_name), - partition_probability_source=str(partition_probability_source), - calibrator_name=str(calibrator_name), + key = (prediction.method, prediction.partition) + cases_by_variant[key] = _make_cases( + y_true=data.y_test, + y_pred=prediction.y_pred, + y_sets=prediction.y_sets, + test_df=data.test_df, + method=prediction.method, + partition=prediction.partition, + partition_probability_source=settings.partition_probability_source, + calibrator_name=data.calibrator_name, + ) + sensitivity_by_variant[key] = _sensitivity_rows_for_variant( + data=data, + settings=settings, + method_name=method_name, + partition_name=prediction.partition, ) - - sensitivity_rows: list[dict[str, Any]] = [] - for frac in calibration_size_fractions: - frac_float = float(frac) - if frac_float <= 0 or frac_float > 1: - continue - cal_df_sub = _subset_calibration_frame(cal_df, calibration_fraction=frac_float) - X_cal_sub = _build_feature_matrix(cal_df_sub, features, categorical) - y_cal_sub = cal_df_sub[TARGET_COL].astype(int).reset_index(drop=True) - group_cal_sub = ( - cal_df_sub[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - ) - y_prob_cal_sub_raw = model.predict_proba(X_cal_sub)[:, 1] - y_prob_cal_sub_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_cal_sub_raw) - if calibrator is not None - else np.asarray(y_prob_cal_sub_raw, dtype=float) - ) - y_prob_cal_sub = ( - y_prob_cal_sub_raw - if partition_probability_source == "raw" - else y_prob_cal_sub_calibrated - ) - if str(partition_name).strip().lower() == "global": - y_pred_sub, y_sets_sub = create_classification_sets( - classifier=model, - X_cal=X_cal_sub, - y_cal=y_cal_sub, - X_test=X_test, - alpha=alpha, - method=method_name, - calibrator=calibrator, - ) - else: - group_cal_part, group_test_part, _ = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal_sub, - y_prob_eval=prob_test_lookup[partition_probability_source], - partition=partition_name, - base_groups_cal=group_cal_sub, - base_groups_eval=group_test, - n_score_bins=n_score_bins, - min_group_size=min_group_size, - fallback_mode=effective_fallback_mode, - ) - y_pred_sub, y_sets_sub, _ = create_classification_sets_mondrian( - classifier=model, - X_cal=X_cal_sub, - y_cal=y_cal_sub, - X_test=X_test, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha, - method=method_name, - min_group_size=min_group_size, - calibrator=calibrator, - ) - summary_sub = summarize_prediction_sets(y_test.to_numpy(), y_pred_sub, y_sets_sub) - sensitivity_rows.append( - { - "method": str(method_name), - "partition": str(partition_name), - "partition_probability_source": str(partition_probability_source), - "calibrator": str(calibrator_name), - "calibration_fraction": frac_float, - "n_calibration_rows": int(len(X_cal_sub)), - "set_coverage": float(summary_sub["set_coverage"]), - "singleton_rate": float(summary_sub["singleton_rate"]), - "ambiguity_rate": float(summary_sub["ambiguity_rate"]), - "empty_set_rate": float(summary_sub["empty_set_rate"]), - "default_rate_ambiguous": float(summary_sub["default_rate_ambiguous"]), - } - ) - sensitivity_by_variant[(str(method_name), str(partition_name))] = sensitivity_rows benchmark_df = pd.DataFrame(benchmark_rows) if benchmark_df.empty: raise RuntimeError("No set-prediction variants were benchmarked.") - benchmark_df = benchmark_df.sort_values( - by=["set_coverage", "singleton_rate", "ambiguity_rate", "empty_set_rate"], - ascending=[False, False, True, True], - ).reset_index(drop=True) - selected = benchmark_df.iloc[0] - selected_key = (str(selected["method"]), str(selected["partition"])) - cases = cases_by_variant[selected_key] + return ( + benchmark_df.sort_values( + by=["set_coverage", "singleton_rate", "ambiguity_rate", "empty_set_rate"], + ascending=[False, False, True, True], + ).reset_index(drop=True), + cases_by_variant, + sensitivity_by_variant, + ) + +def _slice_reports(cases: pd.DataFrame) -> pd.DataFrame: slice_reports = [] for col in ("grade", "term", "issue_quarter"): if col in cases.columns: report = _slice_summary(cases, col) if not report.empty: slice_reports.append(report) - by_slice = pd.concat(slice_reports, ignore_index=True) if slice_reports else pd.DataFrame() - sensitivity_df = pd.DataFrame(sensitivity_by_variant[selected_key]) + return pd.concat(slice_reports, ignore_index=True) if slice_reports else pd.DataFrame() - summary = summarize_prediction_sets( - cases["y_true"].to_numpy(dtype=int), - cases["y_pred_label"].to_numpy(dtype=int), - cases[["set_contains_0", "set_contains_1"]].to_numpy(dtype=int), - ) - grade_slices = ( - by_slice.loc[by_slice["slice_name"] == "grade"].to_dict(orient="records") - if not by_slice.empty and "slice_name" in by_slice.columns - else [] - ) + +def _grade_slices(by_slice: pd.DataFrame) -> list[dict[str, Any]]: + if by_slice.empty or "slice_name" not in by_slice.columns: + return [] + return by_slice.loc[by_slice["slice_name"] == "grade"].to_dict(orient="records") + + +def _slice_records(by_slice: pd.DataFrame, slice_name: str) -> list[dict[str, Any]]: + if by_slice.empty or "slice_name" not in by_slice.columns: + return [] + return by_slice.loc[by_slice["slice_name"] == slice_name].to_dict(orient="records") + + +def _promotion_gate(summary: dict[str, Any], grade_slices: list[dict[str, Any]]) -> dict[str, Any]: gate_coverage = float(summary.get("set_coverage", 0)) gate_grade_a_singleton = 0.0 gate_grades_above_40 = 0 for gs in grade_slices: - sr = float(gs.get("singleton_rate", 0)) + singleton_rate = float(gs.get("singleton_rate", 0)) if str(gs.get("slice_value", "")) == "A": - gate_grade_a_singleton = sr - if sr > 0.40: + gate_grade_a_singleton = singleton_rate + if singleton_rate > 0.40: gate_grades_above_40 += 1 - - gate_pass = ( + gate_pass = bool( gate_coverage >= 0.85 and gate_grade_a_singleton >= 0.80 and gate_grades_above_40 >= 3 ) - promotion_status = "promoted_guardrail" if gate_pass else "research_sidecar" + return { + "coverage": gate_coverage, + "min_coverage": 0.85, + "grade_a_singleton_rate": gate_grade_a_singleton, + "min_grade_a_singleton": 0.80, + "grades_with_singleton_above_40pct": gate_grades_above_40, + "min_grades_above_40pct": 3, + "pass": gate_pass, + } + + +def _selected_summary(cases: pd.DataFrame) -> dict[str, float]: + return { + k: float(v) + for k, v in summarize_prediction_sets( + cases["y_true"].to_numpy(dtype=int), + cases["y_pred_label"].to_numpy(dtype=int), + cases[["set_contains_0", "set_contains_1"]].to_numpy(dtype=int), + ).items() + } - outputs = _build_output_paths(artifact_namespace) - cases.to_parquet(outputs["cases"], index=False) - if not by_slice.empty: - by_slice.to_parquet(outputs["by_slice"], index=False) - sensitivity_df.to_parquet(outputs["sensitivity"], index=False) - benchmark_df.to_parquet(outputs["benchmark"], index=False) - status = { +def _status_payload( + *, + settings: SetBenchmarkSettings, + selected: pd.Series, + summary: dict[str, float], + gate: dict[str, Any], + by_slice: pd.DataFrame, + benchmark_df: pd.DataFrame, + outputs: dict[str, Path], + artifact_namespace: str | None, +) -> dict[str, Any]: + gate_pass = bool(gate["pass"]) + promotion_status = "promoted_guardrail" if gate_pass else "research_sidecar" + return { "schema_version": "2026-04-03.1", "generated_at_utc": datetime.now(tz=UTC).isoformat(), "run_tag": os.environ.get("PIPELINE_RUN_TAG", "untracked"), @@ -422,34 +602,20 @@ def main( "selected_partition": str(selected["partition"]), "selected_partition_probability_source": str(selected["partition_probability_source"]), "selected_calibrator": str(selected["calibrator"]), - "requested_fallback_mode": str(fallback_mode), - "effective_fallback_mode": str(effective_fallback_mode), - "alpha": float(alpha), - "confidence_level": float(1.0 - alpha), - "summary": {k: float(v) for k, v in summary.items()}, - "promotion_gate": { - "coverage": gate_coverage, - "min_coverage": 0.85, - "grade_a_singleton_rate": gate_grade_a_singleton, - "min_grade_a_singleton": 0.80, - "grades_with_singleton_above_40pct": gate_grades_above_40, - "min_grades_above_40pct": 3, - "pass": gate_pass, - }, + "requested_fallback_mode": settings.requested_fallback_mode, + "effective_fallback_mode": settings.effective_fallback_mode, + "alpha": settings.alpha, + "confidence_level": float(1.0 - settings.alpha), + "summary": summary, + "promotion_gate": gate, "artifact_path": str(outputs["cases"]), "by_slice_path": str(outputs["by_slice"]), "calibration_size_sensitivity_path": str(outputs["sensitivity"]), "benchmark_matrix_path": str(outputs["benchmark"]), "slice_metrics": { - "grade": grade_slices, - "term": by_slice.loc[by_slice["slice_name"] == "term"].to_dict(orient="records") - if not by_slice.empty and "slice_name" in by_slice.columns - else [], - "issue_quarter": by_slice.loc[by_slice["slice_name"] == "issue_quarter"].to_dict( - orient="records" - ) - if not by_slice.empty and "slice_name" in by_slice.columns - else [], + "grade": _grade_slices(by_slice), + "term": _slice_records(by_slice, "term"), + "issue_quarter": _slice_records(by_slice, "issue_quarter"), }, "benchmark_matrix": benchmark_df.to_dict(orient="records"), "decision_use_case": { @@ -459,13 +625,29 @@ def main( }, "promotion_rationale": ( f"Binary conformal sets selected via {selected['method']} + {selected['partition']} " - f"with set coverage {gate_coverage:.1%} and ambiguity {summary['ambiguity_rate']:.1%}." + f"with set coverage {gate['coverage']:.1%} and ambiguity {summary['ambiguity_rate']:.1%}." ), "promotion_note": ( "Binary set prediction remains a sidecar triage/abstention signal; it does not replace " "the interval-first conformal stack." ), } + + +def _write_outputs( + *, + outputs: dict[str, Path], + cases: pd.DataFrame, + by_slice: pd.DataFrame, + sensitivity_df: pd.DataFrame, + benchmark_df: pd.DataFrame, + status: dict[str, Any], +) -> None: + cases.to_parquet(outputs["cases"], index=False) + if not by_slice.empty: + by_slice.to_parquet(outputs["by_slice"], index=False) + sensitivity_df.to_parquet(outputs["sensitivity"], index=False) + benchmark_df.to_parquet(outputs["benchmark"], index=False) outputs["status"].write_text( json.dumps(status, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" ) @@ -473,6 +655,64 @@ def main( logger.info("Saved PD set prediction status: {}", outputs["status"]) +def main( + alpha: float = 0.10, + method: str = "lac", + methods: tuple[str, ...] | None = None, + partitions: tuple[str, ...] = ("global",), + partition_probability_source: str = "raw", + n_score_bins: int = 10, + min_group_size: int = 500, + fallback_mode: str = "grade_then_global", + calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), + artifact_namespace: str | None = None, + calibrator_override_path: str | None = None, +) -> None: + data = _load_set_benchmark_data(calibrator_override_path) + settings = _set_benchmark_settings( + alpha=alpha, + method=method, + methods=methods, + partitions=partitions, + partition_probability_source=partition_probability_source, + n_score_bins=n_score_bins, + min_group_size=min_group_size, + fallback_mode=fallback_mode, + calibration_size_fractions=calibration_size_fractions, + prob_cal_lookup=data.prob_cal_lookup, + ) + benchmark_df, cases_by_variant, sensitivity_by_variant = _run_benchmark_matrix( + data=data, + settings=settings, + ) + selected = benchmark_df.iloc[0] + selected_key = (str(selected["method"]), str(selected["partition"])) + cases = cases_by_variant[selected_key] + by_slice = _slice_reports(cases) + sensitivity_df = pd.DataFrame(sensitivity_by_variant[selected_key]) + summary = _selected_summary(cases) + gate = _promotion_gate(summary, _grade_slices(by_slice)) + outputs = _build_output_paths(artifact_namespace) + status = _status_payload( + settings=settings, + selected=selected, + summary=summary, + gate=gate, + by_slice=by_slice, + benchmark_df=benchmark_df, + outputs=outputs, + artifact_namespace=artifact_namespace, + ) + _write_outputs( + outputs=outputs, + cases=cases, + by_slice=by_slice, + sensitivity_df=sensitivity_df, + benchmark_df=benchmark_df, + status=status, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--alpha", type=float, default=0.10) diff --git a/scripts/build_bound_tightening_audit.py b/scripts/build_bound_tightening_audit.py index 65b8b08..3429809 100644 --- a/scripts/build_bound_tightening_audit.py +++ b/scripts/build_bound_tightening_audit.py @@ -335,7 +335,7 @@ def _write_report( ] ) ].sort_values("threshold_t") - lines = [ + lines: list[str] = [ "# CRPTO Bound Tightening Experiment - 2026-06-11", "", "Merged into `main` (2026-06-11) and cited by Online Supplement Appendix A. " @@ -366,26 +366,28 @@ def _write_report( f"`{row.margin_vs_empirical_V:.6f}` | {row.paper_role} |" ) - lines += [ - "", - "## Recommendation", - "", - "- Keep Markov as the body theorem: it is the only first-moment, " - "distribution-free statement compatible with the current post-selection caveat.", - "- Keep A21 cluster-aware Hoeffding as a dependence caveat, not a tightening: " - "cluster exposure is too concentrated.", - "- Use A21b/A21c as an appendix sensitivity table. Cantelli, Bernstein, Bennett " - "and Freedman show how much tightness is available if a reviewer accepts stronger " - "independence, variance, or martingale assumptions.", - "- Drop Chebyshev, Azuma, Chernoff and naive union-Markov from paper-facing tables. " - "They are respectively dominated, duplicative, too strong for the current " - "individual-alpha evidence, or vacuous after policy-region correction.", - "", - "## Assumption Audit", - "", - assumption_table.to_markdown(index=False), - "", - ] + lines.extend( + ( + "", + "## Recommendation", + "", + "- Keep Markov as the body theorem: it is the only first-moment, " + "distribution-free statement compatible with the current post-selection caveat.", + "- Keep A21 cluster-aware Hoeffding as a dependence caveat, not a tightening: " + "cluster exposure is too concentrated.", + "- Use A21b/A21c as an appendix sensitivity table. Cantelli, Bernstein, Bennett " + "and Freedman show how much tightness is available if a reviewer accepts stronger " + "independence, variance, or martingale assumptions.", + "- Drop Chebyshev, Azuma, Chernoff and naive union-Markov from paper-facing tables. " + "They are respectively dominated, duplicative, too strong for the current " + "individual-alpha evidence, or vacuous after policy-region correction.", + "", + "## Assumption Audit", + "", + str(assumption_table.to_markdown(index=False)), + "", + ) + ) REPORT_PATH.parent.mkdir(parents=True, exist_ok=True) REPORT_PATH.write_text("\n".join(lines), encoding="utf-8", newline="") diff --git a/scripts/build_crpto_journal_package.py b/scripts/build_crpto_journal_package.py index 713ce33..02e5e86 100644 --- a/scripts/build_crpto_journal_package.py +++ b/scripts/build_crpto_journal_package.py @@ -156,15 +156,17 @@ def _build_tail_risk_table(funded: pd.DataFrame) -> pd.DataFrame: index=funded.index, ) theta = 5.0 - oce = round(float(entropic_oce(loss_rate, weights, theta=theta, stable=False)), 17) + loss_values = loss_rate.to_numpy(dtype=float) + weight_values = weights.to_numpy(dtype=float) + oce = round(float(entropic_oce(loss_values, weight_values, theta=theta, stable=False)), 17) rows.append( { "lgd": float(lgd), - "mean_loss_rate": float(np.sum(weights * loss_rate)), + "mean_loss_rate": float(np.sum(weight_values * loss_values)), "entropic_oce_theta5": oce, - "cvar_90_loss_rate": weighted_cvar(loss_rate, weights, tail=0.90), - "cvar_95_loss_rate": weighted_cvar(loss_rate, weights, tail=0.95), - "cvar_99_loss_rate": weighted_cvar(loss_rate, weights, tail=0.99), + "cvar_90_loss_rate": weighted_cvar(loss_values, weight_values, tail=0.90), + "cvar_95_loss_rate": weighted_cvar(loss_values, weight_values, tail=0.95), + "cvar_99_loss_rate": weighted_cvar(loss_values, weight_values, tail=0.99), "funded_set_repriced_return": _return_from_frame( funded, weights, diff --git a/scripts/build_ijds_calibration_selected_evidence.py b/scripts/build_ijds_calibration_selected_evidence.py new file mode 100644 index 0000000..685ef0e --- /dev/null +++ b/scripts/build_ijds_calibration_selected_evidence.py @@ -0,0 +1,532 @@ +"""Build paper-facing evidence for the calibration-selected IJDS policy.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from src.optimization.policy_selection import policy_eligibility_mask # noqa: E402 +from src.utils.script_helpers import load_json, write_json, write_table # noqa: E402 + +RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" +EXACT_ALPHA_TAG = "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1" +MODEL_DIR = ROOT / "models/experiments/champion_reopen" / RUN_TAG / "portfolio" +DATA_DIR = ROOT / "data/processed/experiments/champion_reopen" / RUN_TAG / "portfolio" +EXACT_MODEL_DIR = ROOT / "models/experiments/champion_reopen" / EXACT_ALPHA_TAG / "conformal" +TABLE_DIR = ROOT / "reports/crpto/tables" + +TABLE_NAMES = { + "alpha": "crpto_tableA35_exact_alpha_grid", + "selector": "crpto_tableA36_calibration_policy_selector", + "temporal": "crpto_tableA37_calibration_selected_temporal_evaluation", + "grade": "crpto_tableA38_calibration_selected_grade_audit", + "bootstrap": "crpto_tableA39_calibration_selected_bootstrap", + "baseline": "crpto_tableA40_calibration_selected_point_baseline", +} +GOVERNANCE_NAME = "ijds_policy_governance.json" + + +def _weighted_average(frame: pd.DataFrame, column: str) -> float: + weights = frame["funded_weight"].to_numpy(dtype=float) + values = frame[column].to_numpy(dtype=float) + return float(np.sum(weights * values) / max(weights.sum(), 1e-12)) + + +def build_alpha_table(summary: dict[str, Any]) -> pd.DataFrame: + rows = pd.DataFrame(summary["alpha_summaries"]).copy() + rows.insert(0, "selected_for_policy", np.isclose(rows["target_alpha"], 0.10)) + return rows[ + [ + "selected_for_policy", + "target_alpha", + "used_alpha", + "target_coverage", + "empirical_coverage", + "coverage_gap", + "avg_width", + "min_partition_coverage", + "min_grade_coverage", + "high_endpoint_at_one_rate", + ] + ] + + +def build_selector_table( + grid: pd.DataFrame, + audit_grid: pd.DataFrame, + summary: dict[str, Any], +) -> pd.DataFrame: + selected_id = str(summary["selected_policy"]["candidate_id"]) + audit_selected_id = str(summary["calibration_audit"]["outcome_free_selected_candidate_id"]) + cap = float(summary["design"]["endpoint_budget_cap"]) + output = grid.copy() + output["eligible"] = policy_eligibility_mask( + output, + endpoint_budget_cap=cap, + budget=float(summary["design"]["budget"]), + min_budget_utilization=float(summary["design"]["selection_min_budget_utilization"]), + ) + output["selected"] = output["candidate_id"].astype(str).eq(selected_id) + audit = audit_grid.copy() + audit["audit_eligible"] = policy_eligibility_mask( + audit, + endpoint_budget_cap=cap, + budget=float(summary["design"]["budget"]), + min_budget_utilization=float(summary["design"]["selection_min_budget_utilization"]), + ) + audit["audit_selected"] = audit["candidate_id"].astype(str).eq(audit_selected_id) + audit = audit.rename( + columns={ + "expected_objective": "audit_expected_objective", + "n_funded": "audit_n_funded", + "endpoint_budget": "audit_endpoint_budget", + } + ) + output = output.merge( + audit[ + [ + "candidate_id", + "audit_selected", + "audit_eligible", + "audit_expected_objective", + "audit_n_funded", + "audit_endpoint_budget", + ] + ], + on="candidate_id", + how="left", + validate="one_to_one", + ) + return output[ + [ + "selected", + "eligible", + "audit_selected", + "audit_eligible", + "candidate_id", + "risk_tolerance", + "gamma", + "expected_objective", + "n_funded", + "weighted_pd_point", + "weighted_pd_effective", + "endpoint_budget", + "audit_expected_objective", + "audit_n_funded", + "audit_endpoint_budget", + ] + ].sort_values( + ["selected", "eligible", "expected_objective"], + ascending=[False, False, False], + kind="mergesort", + ) + + +def build_temporal_table(evaluation: pd.DataFrame) -> pd.DataFrame: + role_labels = { + "calibration_selected": "Calibration-selected 50/50 CRPTO", + "incumbent_linear": "More-conservative 75% blend", + "point_pd_matched_tau": "Point-PD matched-tau", + } + output = evaluation.copy() + output["policy"] = output["role"].map(role_labels) + return output[ + [ + "period", + "policy", + "risk_tolerance", + "gamma", + "n_funded", + "expected_objective", + "realized_return", + "weighted_outcome", + "weighted_miscoverage", + "weighted_pd_point", + "weighted_pd_effective", + "endpoint_budget", + "markov_loss_threshold", + ] + ] + + +def build_grade_table(allocations: pd.DataFrame) -> pd.DataFrame: + selected = allocations.loc[allocations["role"].eq("calibration_selected")].copy() + rows: list[dict[str, Any]] = [] + for grade, group in selected.groupby("grade", observed=True, sort=True): + rows.append( + { + "grade": str(grade), + "n_funded": int(len(group)), + "exposure": float(group["funded_exposure"].sum()), + "exposure_share": float(group["funded_weight"].sum()), + "weighted_default_rate": _weighted_average(group, "outcome"), + "weighted_miscoverage": _weighted_average(group, "miscoverage"), + "weighted_pd_point": _weighted_average(group, "pd_point"), + "weighted_pd_effective": _weighted_average(group, "pd_effective"), + "weighted_pd_high": _weighted_average(group, "pd_high"), + "realized_return": float(group["realized_return_contribution"].sum()), + } + ) + return pd.DataFrame(rows) + + +def _bootstrap_snapshot( + sample: pd.DataFrame, *, total_exposure: float, lgd: float +) -> dict[str, float]: + weights = sample["funded_exposure"].to_numpy(dtype=float) + weights = weights / max(float(weights.sum()), 1e-12) + outcome = sample["outcome"].to_numpy(dtype=float) + rates = sample["int_rate"].to_numpy(dtype=float) + point = sample["pd_point"].to_numpy(dtype=float) + high = sample["pd_high"].to_numpy(dtype=float) + realized_rate = np.where(outcome.astype(int) == 1, -float(lgd), rates) + return { + "realized_return": float(np.sum(weights * realized_rate) * total_exposure), + "weighted_default_rate": float(np.sum(weights * outcome)), + "weighted_miscoverage": float(np.sum(weights * sample["miscoverage"])), + "Gamma_CP": float(np.sum(weights * (high - point))), + "endpoint_budget": float(np.sum(weights * high)), + } + + +def _bootstrap_summary_rows( + draw_frame: pd.DataFrame, + observed: dict[str, float], + *, + bootstrap_unit: str, + n_units: int, + n_draws: int, + seed: int, + note: str, +) -> list[dict[str, Any]]: + return [ + { + "bootstrap_unit": bootstrap_unit, + "metric": metric, + "observed": observed[metric], + "boot_mean": float(draw_frame[metric].mean()), + "boot_p025": float(draw_frame[metric].quantile(0.025)), + "boot_p50": float(draw_frame[metric].quantile(0.50)), + "boot_p975": float(draw_frame[metric].quantile(0.975)), + "n_units": n_units, + "n_draws": n_draws, + "seed": seed, + "note": note, + } + for metric in draw_frame.columns + ] + + +def build_bootstrap_table( + allocations: pd.DataFrame, + evaluation: pd.DataFrame, + *, + n_draws: int = 5000, + seed: int = 20260709, + lgd: float = 0.45, +) -> pd.DataFrame: + selected = allocations.loc[allocations["role"].eq("calibration_selected")].reset_index( + drop=True + ) + total_exposure = float(selected["funded_exposure"].sum()) + allocation_observed = _bootstrap_snapshot( + selected, + total_exposure=total_exposure, + lgd=lgd, + ) + official = evaluation.loc[ + evaluation["period"].eq("full_oot") & evaluation["role"].eq("calibration_selected") + ].iloc[0] + observed = { + "realized_return": float(official["realized_return"]), + "weighted_default_rate": float(official["weighted_outcome"]), + "weighted_miscoverage": float(official["weighted_miscoverage"]), + "Gamma_CP": float(official["gamma_cp"]), + "endpoint_budget": float(official["endpoint_budget"]), + } + mismatches = [ + metric + for metric, value in observed.items() + if not np.isclose(allocation_observed[metric], value, rtol=1e-10, atol=1e-8) + ] + if mismatches: + raise ValueError( + "Funded allocations do not reconcile to the full-OOT evaluation: " + + ", ".join(mismatches) + ) + loan_rng = np.random.default_rng(seed) + loan_draws = [ + _bootstrap_snapshot( + selected.iloc[loan_rng.integers(0, len(selected), size=len(selected))].reset_index( + drop=True + ), + total_exposure=total_exposure, + lgd=lgd, + ) + for _ in range(n_draws) + ] + issue_month = pd.to_datetime(selected["issue_d"], errors="raise").dt.to_period("M") + month_groups = [ + selected.loc[issue_month.eq(month)].reset_index(drop=True) + for month in sorted(issue_month.unique()) + ] + month_rng = np.random.default_rng(seed) + month_draws = [ + _bootstrap_snapshot( + pd.concat( + [ + month_groups[index] + for index in month_rng.integers(0, len(month_groups), len(month_groups)) + ], + ignore_index=True, + ), + total_exposure=total_exposure, + lgd=lgd, + ) + for _ in range(n_draws) + ] + rows = _bootstrap_summary_rows( + pd.DataFrame(month_draws), + observed, + bootstrap_unit="origination_month", + n_units=len(month_groups), + n_draws=n_draws, + seed=seed, + note=( + "Fixed-allocation origination-month cluster bootstrap, renormalized to the " + "$1M budget; model, intervals, selector, and solver are not resampled." + ), + ) + rows.extend( + _bootstrap_summary_rows( + pd.DataFrame(loan_draws), + observed, + bootstrap_unit="funded_loan", + n_units=len(selected), + n_draws=n_draws, + seed=seed, + note=( + "Fixed funded-loan contribution bootstrap; model, intervals, selector, " + "and solver are not resampled." + ), + ) + ) + return pd.DataFrame(rows) + + +def build_baseline_table(evaluation: pd.DataFrame) -> pd.DataFrame: + full = evaluation.loc[evaluation["period"].eq("full_oot")].copy() + selected = full.loc[full["role"].eq("calibration_selected")].iloc[0] + labels = { + "calibration_selected": "Calibration-selected 50/50 CRPTO", + "incumbent_linear": "More-conservative 75% blend", + "point_pd_matched_tau": "Point-PD matched-tau", + } + full["policy"] = full["role"].map(labels) + full["return_delta_vs_selected"] = full["realized_return"] - float(selected["realized_return"]) + full["default_delta_vs_selected"] = full["weighted_outcome"] - float( + selected["weighted_outcome"] + ) + full["threshold_delta_vs_selected"] = full["markov_loss_threshold"] - float( + selected["markov_loss_threshold"] + ) + return full[ + [ + "policy", + "n_funded", + "expected_objective", + "realized_return", + "weighted_outcome", + "weighted_miscoverage", + "endpoint_budget", + "markov_loss_threshold", + "return_delta_vs_selected", + "default_delta_vs_selected", + "threshold_delta_vs_selected", + ] + ] + + +def _compact_calibration_audit(summary: dict[str, Any]) -> dict[str, Any]: + audit = summary["calibration_audit"] + rows = {str(row["role"]): row for row in audit["policy_evaluations"]} + + def _policy(role: str) -> dict[str, Any]: + row = rows[role] + return { + "n_funded": int(row["n_funded"]), + "realized_return": float(row["realized_return"]), + "weighted_default_rate": float(row["weighted_outcome"]), + "weighted_miscoverage": float(row["weighted_miscoverage"]), + "endpoint_budget": float(row["endpoint_budget"]), + "observed_accounting_bound": float( + row["endpoint_budget"] + row["weighted_miscoverage"] + ), + } + + return { + "period": audit["period"], + "n_candidates": int(summary["calibration_metadata"]["audit_rows"]), + "outcome_free_selected_candidate_id": audit["outcome_free_selected_candidate_id"], + "same_policy_selected": bool(audit["same_policy_selected"]), + "selected_policy": _policy("calibration_selected"), + "more_conservative_policy": _policy("incumbent_linear"), + "matched_point_pd": _policy("point_pd_matched_tau"), + "claim_boundary": audit["claim_boundary"], + } + + +def build_governance( + summary: dict[str, Any], + exact_summary: dict[str, Any], + evaluation: pd.DataFrame, + bootstrap: pd.DataFrame, + table_paths: dict[str, list[Path]], +) -> dict[str, Any]: + full = evaluation.loc[ + evaluation["period"].eq("full_oot") & evaluation["role"].eq("calibration_selected") + ].iloc[0] + point = evaluation.loc[ + evaluation["period"].eq("full_oot") & evaluation["role"].eq("point_pd_matched_tau") + ].iloc[0] + return_boot = bootstrap.loc[ + bootstrap["metric"].eq("realized_return") + & bootstrap["bootstrap_unit"].eq("origination_month") + ].iloc[0] + return { + "schema_version": "2026-07-09.7", + "generated_at_utc": summary["generated_at_utc"], + "run_tag": RUN_TAG, + "status": "active_ijds_policy", + "selection_protocol": { + **summary["selection_audit"], + "calibration_metadata": summary["calibration_metadata"], + "selector_forbidden_columns_present": summary["selector_forbidden_columns_present"], + "selector_input_columns": summary["selector_input_columns"], + "endpoint_cap_stability": summary["endpoint_cap_stability"], + "calibration_audit": _compact_calibration_audit(summary), + }, + "selected_policy": summary["selected_policy"], + "full_oot": { + "n_candidates": int(full["n_panel"]), + "n_funded": int(full["n_funded"]), + "total_allocated": float(full["total_allocated"]), + "expected_objective": float(full["expected_objective"]), + "realized_return": float(full["realized_return"]), + "weighted_default_rate": float(full["weighted_outcome"]), + "weighted_miscoverage": float(full["weighted_miscoverage"]), + "weighted_pd_point": float(full["weighted_pd_point"]), + "weighted_pd_effective": float(full["weighted_pd_effective"]), + "Gamma_CP": float(full["gamma_cp"]), + "Gamma_internalized": float(full["gamma_internalized"]), + "Gamma_residual": float(full["gamma_residual"]), + "endpoint_budget": float(full["endpoint_budget"]), + "markov_loss_threshold": float(full["markov_loss_threshold"]), + "observed_accounting_bound": float( + full["endpoint_budget"] + full["weighted_miscoverage"] + ), + "markov_tail_probability_bound": float(np.sqrt(summary["design"]["alpha"])), + }, + "point_pd_contrast": { + "realized_return": float(point["realized_return"]), + "weighted_default_rate": float(point["weighted_outcome"]), + "weighted_miscoverage": float(point["weighted_miscoverage"]), + "endpoint_budget": float(point["endpoint_budget"]), + "markov_loss_threshold": float(point["markov_loss_threshold"]), + "selected_return_cost": float(point["realized_return"] - full["realized_return"]), + "selected_return_cost_pct": float( + (point["realized_return"] - full["realized_return"]) / point["realized_return"] + ), + "selected_default_reduction": float( + point["weighted_outcome"] - full["weighted_outcome"] + ), + "selected_threshold_reduction": float( + point["markov_loss_threshold"] - full["markov_loss_threshold"] + ), + }, + "bootstrap_return_interval": { + "bootstrap_unit": "origination_month", + "p025": float(return_boot["boot_p025"]), + "p975": float(return_boot["boot_p975"]), + "n_units": int(return_boot["n_units"]), + "n_draws": int(return_boot["n_draws"]), + }, + "exact_alpha_reference_replay": exact_summary["reference_replay"], + "paper_tables": { + key: [str(path.relative_to(ROOT).as_posix()) for path in paths] + for key, paths in table_paths.items() + }, + "retired_active_claims": [ + "alpha01 intervals obtained by cross-family average-width scaling", + "8/8 approximate alpha-grid pass as a headline certificate", + "50,010-policy frontier as the active selector", + "0.345084 Markov threshold", + "capped_blended_uncertainty with delta_cap_quantile=0.975", + "OOT-outcome-selected portfolio hyperparameters", + "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger", + ], + "claim_boundary": summary["claim_boundary"], + } + + +def run(*, bootstrap_draws: int, bootstrap_seed: int) -> dict[str, Any]: + summary = load_json(MODEL_DIR / "calibration_selected_policy_summary.json") + exact_summary = load_json(EXACT_MODEL_DIR / "exact_alpha_grid_summary.json") + grid = pd.read_parquet(DATA_DIR / "calibration_policy_selection_grid.parquet") + audit_grid = pd.read_parquet(DATA_DIR / "calibration_policy_audit_grid.parquet") + evaluation = pd.read_csv(DATA_DIR / "calibration_selected_policy_oot_evaluation.csv") + allocations = pd.read_parquet( + DATA_DIR / "calibration_selected_policy_full_oot_allocations.parquet" + ) + tables = { + "alpha": build_alpha_table(exact_summary), + "selector": build_selector_table(grid, audit_grid, summary), + "temporal": build_temporal_table(evaluation), + "grade": build_grade_table(allocations), + "bootstrap": build_bootstrap_table( + allocations, + evaluation, + n_draws=bootstrap_draws, + seed=bootstrap_seed, + ), + "baseline": build_baseline_table(evaluation), + } + table_paths = { + key: write_table(TABLE_NAMES[key], frame, table_dir=TABLE_DIR, root=ROOT) + for key, frame in tables.items() + } + governance = build_governance( + summary, + exact_summary, + evaluation, + tables["bootstrap"], + table_paths, + ) + write_json(MODEL_DIR / GOVERNANCE_NAME, governance) + return governance + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--bootstrap-draws", type=int, default=5000) + parser.add_argument("--bootstrap-seed", type=int, default=20260709) + args = parser.parse_args() + payload = run( + bootstrap_draws=max(100, int(args.bootstrap_draws)), + bootstrap_seed=int(args.bootstrap_seed), + ) + print(json.dumps(payload["full_oot"], indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_papers_tesis_deep_audit.py b/scripts/build_papers_tesis_deep_audit.py index 94ce9aa..ebed162 100644 --- a/scripts/build_papers_tesis_deep_audit.py +++ b/scripts/build_papers_tesis_deep_audit.py @@ -16,6 +16,7 @@ from collections import Counter from dataclasses import dataclass, field from pathlib import Path +from typing import Any DEFAULT_SOURCE_DIR = Path("Papers_tesis") DEFAULT_AUDIT_PATH = Path("docs/research/papers_tesis_deep_audit_2026-06-06.md") @@ -59,8 +60,8 @@ class Review: tags: tuple[str, ...] = field(default_factory=tuple) -def r(**kwargs: object) -> Review: - return Review(**kwargs) # type: ignore[arg-type] +def r(**kwargs: Any) -> Review: + return Review(**kwargs) REVIEWS: dict[str, Review] = { @@ -1772,191 +1773,235 @@ def markdown_table( return "\n".join([header, divider, *body]) -def write_audit( - path: Path, - rows: list[dict[str, object]], - matrix_path: Path, - caption_path: Path, - curated_visual_path: Path, - curated_visual_rows: list[dict[str, object]], -) -> None: - path.parent.mkdir(parents=True, exist_ok=True) +def _counter_rows(counter: Counter[str], key_field: str) -> list[dict[str, object]]: + return [{key_field: key, "n": value} for key, value in sorted(counter.items())] + + +def _section(title: str, *body: str) -> list[str]: + return ["", f"## {title}", "", *body] + + +def _decision_groups(rows: list[dict[str, object]]) -> dict[str, list[dict[str, object]]]: + return { + "promote": [row for row in rows if str(row["decision"]).startswith("promote")], + "crpto_appendix": [ + row + for row in rows + if str(row["decision"]).startswith(("append_crpto", "append_comparator", "append_tail")) + ], + "extended": [ + row + for row in rows + if "extended" in str(row["decision"]) or "mixed_diagnostic" in str(row["decision"]) + ], + "future": [ + row + for row in rows + if "future" in str(row["decision"]) or "candidate" in str(row["decision"]) + ], + "experiments": [ + row for row in rows if str(row["action_required"]).startswith("experiment_") + ], + } + + +def _inventory_sections(rows: list[dict[str, object]]) -> list[str]: folder_counts = Counter(str(row["folder"]) for row in rows) decision_counts = Counter(str(row["decision"]) for row in rows) action_counts = Counter(str(row["action_required"]) for row in rows) domain_counts = Counter(str(row["primary_domain"]) for row in rows) - - promote = [row for row in rows if str(row["decision"]).startswith("promote")] - crpto_appendix = [ - row - for row in rows - if str(row["decision"]).startswith(("append_crpto", "append_comparator", "append_tail")) - ] - extended = [ - row - for row in rows - if "extended" in str(row["decision"]) or "mixed_diagnostic" in str(row["decision"]) - ] - future = [ - row - for row in rows - if "future" in str(row["decision"]) or "candidate" in str(row["decision"]) - ] - - lines = [ - "# Papers_tesis Deep Audit - 2026-06-06", - "", - "## Resumen ejecutivo", - "", - f"Esta auditoría cubre `61` PDFs locales en `{DEFAULT_SOURCE_DIR}` y fue generada para el corte `{REPORT_DATE}` con `scripts/build_papers_tesis_deep_audit.py`.", - "", - "La decisión central no cambia: **Paper CRPTO conserva el champion oficial** y la literatura nueva se usa para reforzar teoría, related work, appendices y límites de claim. La agenda extendida CRPTO/tesis absorbe el material que sí pertenece al laboratorio vivo: source/shift conformal, utility-directed conformal, tail risk, DFL, IFRS9 proxy, data/noise/equity y governance.", - "", - "Artefactos generados:", - "", - f"- Matriz fuente: `{matrix_path}`", - f"- Índice compacto de captions: `{caption_path}`", - f"- Curaduría de visual sinks: `{curated_visual_path}`", - "", - "## Inventario", - "", - markdown_table( - [{"folder": folder, "pdfs": count} for folder, count in sorted(folder_counts.items())], - ["folder", "pdfs"], + return [ + *_section( + "Inventario", + markdown_table( + [ + {"folder": folder, "pdfs": count} + for folder, count in sorted(folder_counts.items()) + ], + ["folder", "pdfs"], + ), ), - "", - "## Decisiones por destino editorial", - "", - markdown_table( - [{"decision": key, "n": value} for key, value in sorted(decision_counts.items())], - ["decision", "n"], + *_section( + "Decisiones por destino editorial", + markdown_table(_counter_rows(decision_counts, "decision"), ["decision", "n"]), ), - "", - "## Acciones requeridas", - "", - markdown_table( - [{"action_required": key, "n": value} for key, value in sorted(action_counts.items())], - ["action_required", "n"], + *_section( + "Acciones requeridas", + markdown_table( + _counter_rows(action_counts, "action_required"), ["action_required", "n"] + ), ), - "", - "## Familias conceptuales", - "", - markdown_table( - [{"primary_domain": key, "n": value} for key, value in sorted(domain_counts.items())], - ["primary_domain", "n"], + *_section( + "Familias conceptuales", + markdown_table(_counter_rows(domain_counts, "primary_domain"), ["primary_domain", "n"]), ), - "", - "## Lectura integrada para Paper CRPTO", - "", - "El cuerpo de Paper CRPTO debe quedarse en cuatro pilares: conformal risk/control, robust optimization, conformal robust optimization / predict-then-calibrate y contexto Lending Club/DFL como comparador. Las fuentes promovidas al cuerpo son:", - "", - markdown_table( - promote, ["relative_path", "bib_key", "core_concepts", "crpto_value", "limitations"] + ] + + +def _paper_crpto_sections(groups: dict[str, list[dict[str, object]]]) -> list[str]: + return [ + *_section( + "Lectura integrada para Paper CRPTO", + "El cuerpo de Paper CRPTO debe quedarse en cuatro pilares: conformal risk/control, robust optimization, conformal robust optimization / predict-then-calibrate y contexto Lending Club/DFL como comparador. Las fuentes promovidas al cuerpo son:", + "", + markdown_table( + groups["promote"], + ["relative_path", "bib_key", "core_concepts", "crpto_value", "limitations"], + ), + "", + "Las fuentes de appendix o comparador CRPTO deben apoyar selectivamente el selector, SPO+/DFL, CVaR/OCE, CQR y limites de claim:", + "", + markdown_table( + groups["crpto_appendix"], + ["relative_path", "bib_key", "decision", "crpto_value", "figures_tables_useful"], + ), ), - "", - "Las fuentes de appendix o comparador CRPTO deben apoyar selectivamente el selector, SPO+/DFL, CVaR/OCE, CQR y límites de claim:", - "", - markdown_table( - crpto_appendix, - ["relative_path", "bib_key", "decision", "crpto_value", "figures_tables_useful"], + *_section( + "Lectura integrada para agenda extendida CRPTO/tesis", + "La agenda extendida CRPTO/tesis es el destino correcto para fuentes que fortalecen governance, source/shift robustness, fairness proxy, IFRS9/SICR proxy, DFL ampliado y data-quality/equity. Estas fuentes no reabren el champion CRPTO:", + "", + markdown_table( + groups["extended"], + ["relative_path", "decision", "extended_lab_value", "evidence_gate", "stop_rule"], + ), ), - "", - "## Lectura integrada para agenda extendida CRPTO/tesis", - "", - "La agenda extendida CRPTO/tesis es el destino correcto para fuentes que fortalecen governance, source/shift robustness, fairness proxy, IFRS9/SICR proxy, DFL ampliado y data-quality/equity. Estas fuentes no reabren el champion CRPTO:", - "", - markdown_table( - extended, - ["relative_path", "decision", "extended_lab_value", "evidence_gate", "stop_rule"], + *_section( + "Experimentos evidence-gated", + "Esta seccion lista los experimentos ya ejecutados o pendientes bajo regla evidence-gated. Cada fila exige claim target, evidence gate, artifact sink y stop rule; un resultado positivo no cambia el champion CRPTO sin gate editorial separado.", + "", + markdown_table( + groups["experiments"], + [ + "relative_path", + "action_required", + "implementation_or_experiment", + "evidence_gate", + "artifact_sink", + "stop_rule", + ], + ), ), - "", - "## Experimentos evidence-gated", - "", - "Esta sección lista los experimentos ya ejecutados o pendientes bajo regla evidence-gated. Cada fila exige claim target, evidence gate, artifact sink y stop rule; un resultado positivo no cambia el champion CRPTO sin gate editorial separado.", - "", - markdown_table( - [row for row in rows if str(row["action_required"]).startswith("experiment_")], - [ - "relative_path", - "action_required", - "implementation_or_experiment", - "evidence_gate", - "artifact_sink", - "stop_rule", - ], + ] + + +def _visual_and_matrix_sections( + rows: list[dict[str, object]], + groups: dict[str, list[dict[str, object]]], + curated_visual_rows: list[dict[str, object]], +) -> list[str]: + return [ + *_section( + "Curaduria de figuras/tablas", + "El indice de captions no autoriza reproducir figuras ajenas ni convierte resultados externos en evidencia del proyecto. La curaduria siguiente solo define que visuales pueden inspirar tablas, esquemas propios, appendices o respuestas a reviewers.", + "", + markdown_table( + curated_visual_rows, + [ + "relative_path", + "caption_type", + "caption_index", + "editorial_sink", + "why_useful", + "claim_boundary", + ], + ), ), - "", - "## Curaduría de figuras/tablas", - "", - "El índice de captions no autoriza reproducir figuras ajenas ni convierte resultados externos en evidencia del proyecto. La curaduría siguiente solo define qué visuales pueden inspirar tablas, esquemas propios, appendices o respuestas a reviewers.", - "", - markdown_table( - curated_visual_rows, - [ - "relative_path", - "caption_type", - "caption_index", - "editorial_sink", - "why_useful", - "claim_boundary", - ], + *_section( + "Future work y stop rules", + markdown_table( + groups["future"], + ["relative_path", "decision", "crpto_value", "extended_lab_value", "stop_rule"], + ), ), - "", - "## Future work y stop rules", - "", - markdown_table( - future, ["relative_path", "decision", "crpto_value", "extended_lab_value", "stop_rule"] + *_section( + "Matriz paper-by-paper", + "La tabla siguiente es deliberadamente densa. Cada fila resume concepto, claim, metodo/evidencia, conclusion, figuras/tablas utiles, limitacion y destino editorial. Para auditoria operativa usar el CSV completo.", + "", + markdown_table( + rows, + [ + "relative_path", + "title", + "status", + "core_concepts", + "key_claims", + "conclusions", + "figures_tables_useful", + "limitations", + "decision", + "action_required", + ], + ), ), - "", - "## Matriz paper-by-paper", - "", - "La tabla siguiente es deliberadamente densa. Cada fila resume concepto, claim, método/evidencia, conclusión, figuras/tablas útiles, limitación y destino editorial. Para auditoría operativa usar el CSV completo.", - "", - markdown_table( - rows, - [ - "relative_path", - "title", - "status", - "core_concepts", - "key_claims", - "conclusions", - "figures_tables_useful", - "limitations", - "decision", - "action_required", - ], + ] + + +def _bibliography_and_closeout_sections(rows: list[dict[str, object]]) -> list[str]: + bib_counts = Counter(str(row["bib_status"]) for row in rows) + return [ + *_section( + "Control bibliografico", + "Regla aplicada: `book/references.bib` se modifica solo cuando una fuente queda citada o se prepara explicitamente para texto Quarto de Paper CRPTO/agenda extendida CRPTO/tesis. Las fuentes `needs_bib_if_cited` permanecen en la matriz sin inflar la bibliografia.", + "", + markdown_table(_counter_rows(bib_counts, "bib_status"), ["bib_status", "n"]), ), - "", - "## Control bibliográfico", - "", - "Regla aplicada: `book/references.bib` se modifica solo cuando una fuente queda citada o se prepara explícitamente para texto Quarto de Paper CRPTO/agenda extendida CRPTO/tesis. Las fuentes `needs_bib_if_cited` permanecen en la matriz sin inflar la bibliografía.", - "", - markdown_table( - [ - { - "bib_status": key, - "n": value, - } - for key, value in sorted(Counter(str(row["bib_status"]) for row in rows).items()) - ], - ["bib_status", "n"], + *_section( + "Fronteras que permanecen falsas", + "- CRPTO no reclama legal fair lending con atributos protegidos directos.", + "- CRPTO no implementa IFRS9 contractual.", + "- agenda extendida CRPTO/tesis no reclama CATE policy value.", + "- agenda extendida CRPTO/tesis no reclama online deployment.", + "- agenda extendida CRPTO/tesis no reclama Bellman/DLA exacto.", + "- SPO+/DFL puede ganar regret, pero no reemplaza la garantia/auditabilidad CRPTO.", + ), + *_section( + "Cierre", + "La auditoria agrega valor como integracion bibliografica y de claims. No crea un nuevo champion, no exige nuevas corridas y no transforma fuentes future-work en evidencia empirica del paper actual.", ), + ] + + +def _audit_lines( + rows: list[dict[str, object]], + matrix_path: Path, + caption_path: Path, + curated_visual_path: Path, + curated_visual_rows: list[dict[str, object]], +) -> list[str]: + groups = _decision_groups(rows) + lines = [ + "# Papers_tesis Deep Audit - 2026-06-06", "", - "## Fronteras que permanecen falsas", + "## Resumen ejecutivo", "", - "- CRPTO no reclama legal fair lending con atributos protegidos directos.", - "- CRPTO no implementa IFRS9 contractual.", - "- agenda extendida CRPTO/tesis no reclama CATE policy value.", - "- agenda extendida CRPTO/tesis no reclama online deployment.", - "- agenda extendida CRPTO/tesis no reclama Bellman/DLA exacto.", - "- SPO+/DFL puede ganar regret, pero no reemplaza la garantía/auditabilidad CRPTO.", + f"Esta auditoria cubre `61` PDFs locales en `{DEFAULT_SOURCE_DIR}` y fue generada para el corte `{REPORT_DATE}` con `scripts/build_papers_tesis_deep_audit.py`.", "", - "## Cierre", + "La decision central no cambia: **Paper CRPTO conserva el champion oficial** y la literatura nueva se usa para reforzar teoria, related work, appendices y limites de claim. La agenda extendida CRPTO/tesis absorbe el material que si pertenece al laboratorio vivo: source/shift conformal, utility-directed conformal, tail risk, DFL, IFRS9 proxy, data/noise/equity y governance.", "", - "La auditoría agrega valor como integración bibliográfica y de claims. No crea un nuevo champion, no exige nuevas corridas y no transforma fuentes future-work en evidencia empírica del paper actual.", + "Artefactos generados:", "", + f"- Matriz fuente: `{matrix_path}`", + f"- Indice compacto de captions: `{caption_path}`", + f"- Curaduria de visual sinks: `{curated_visual_path}`", ] + lines.extend(_inventory_sections(rows)) + lines.extend(_paper_crpto_sections(groups)) + lines.extend(_visual_and_matrix_sections(rows, groups, curated_visual_rows)) + lines.extend(_bibliography_and_closeout_sections(rows)) + lines.append("") + return lines + + +def write_audit( + path: Path, + rows: list[dict[str, object]], + matrix_path: Path, + caption_path: Path, + curated_visual_path: Path, + curated_visual_rows: list[dict[str, object]], +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + lines = _audit_lines(rows, matrix_path, caption_path, curated_visual_path, curated_visual_rows) path.write_text("\n".join(lines), encoding="utf-8") diff --git a/scripts/build_tail_constrained_reoptimization.py b/scripts/build_tail_constrained_reoptimization.py index ea431d9..f7d6163 100644 --- a/scripts/build_tail_constrained_reoptimization.py +++ b/scripts/build_tail_constrained_reoptimization.py @@ -294,10 +294,10 @@ def build_tail_constrained_reoptimization(max_policies: int = 0) -> dict[str, An champion_return = float(promotion["final_champion"]["realized_total_return"]) scored_rows: list[dict[str, Any]] = [] - for idx, row in shortlist.iterrows(): + for position, row in enumerate((row for _, row in shortlist.iterrows()), start=1): logger.info( "Re-solving policy {}/{} (candidate_rank={}, mode={}, gamma={})", - idx + 1, + position, len(shortlist), row["candidate_rank"], row["policy_mode"], diff --git a/scripts/build_tail_satisficing_challenger_audit.py b/scripts/build_tail_satisficing_challenger_audit.py index a53bcc2..94ecf1b 100644 --- a/scripts/build_tail_satisficing_challenger_audit.py +++ b/scripts/build_tail_satisficing_challenger_audit.py @@ -11,7 +11,7 @@ import math import sys from pathlib import Path -from typing import Any +from typing import Any, cast import numpy as np import pandas as pd @@ -29,6 +29,7 @@ _solve_single, ) from src.optimization.tail_satisficing_objective import ( # noqa: E402 + SatisficingSense, SatisficingThreshold, entropic_oce, funded_loss_rate, @@ -150,11 +151,16 @@ def _thresholds_from_config(config: dict[str, Any]) -> tuple[SatisficingThreshol raw = config.get("satisficing_thresholds", {}) thresholds: list[SatisficingThreshold] = [] for metric, spec in raw.items(): + sense = str(spec["sense"]) + if sense not in {"min", "max", "equals"}: + raise ValueError(f"Unsupported satisficing threshold sense: {sense!r}") thresholds.append( SatisficingThreshold( metric=str(metric), - sense=str(spec["sense"]), # type: ignore[arg-type] - threshold=spec["threshold"], + sense=cast(SatisficingSense, sense), + threshold=float(spec["threshold"]) + if not isinstance(spec["threshold"], bool) + else spec["threshold"], ) ) return tuple(thresholds) @@ -312,10 +318,10 @@ def _build_a20_table() -> tuple[pd.DataFrame, dict[str, Any]]: loans, pd_point, pd_low, pd_high, lgd, int_rates, default_flag = _prepare_portfolio_inputs() rows: list[dict[str, Any]] = [] - for idx, row in shortlist.iterrows(): + for position, row in enumerate((row for _, row in shortlist.iterrows()), start=1): logger.info( "Auditing policy {}/{} (candidate_rank={})", - idx + 1, + position, len(shortlist), row["candidate_rank"], ) diff --git a/scripts/check_publication_integrity.py b/scripts/check_publication_integrity.py new file mode 100644 index 0000000..3579b06 --- /dev/null +++ b/scripts/check_publication_integrity.py @@ -0,0 +1,190 @@ +"""Check active IJDS manuscript surfaces for claim and narrative drift.""" + +from __future__ import annotations + +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +from loguru import logger + +REPO = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class SurfaceCheck: + """Text surface and tokens that must or must not appear after normalization.""" + + path: Path + required: tuple[str, ...] + forbidden: tuple[str, ...] = () + + +COMMON_CLAIM_TOKENS = ( + "$179327.59", + "0.039375", + "0.036875", + "0.176102", + "0.088051", + "0.258051", + "0.294926", + "0.574279", + "196369.14", + "8.678%", + "7.9025", +) + +ACTIVE_SURFACE_FORBIDDEN = ( + "four contributions", + "crpto v2", + "-10.56%", + "markov cap", + "0.345084", + "50010", + "27508", + "capped_blended_uncertainty", +) + +SURFACES = ( + SurfaceCheck( + path=REPO / "README.md", + required=( + *COMMON_CLAIM_TOKENS, + "claim ijds activo", + "q=(p+u)/2", + "cap determinista", + ), + forbidden=("## champion congelado",), + ), + SurfaceCheck( + path=REPO / "paper/submission/README.md", + required=( + "pdflatex -> bibtex -> pdflatex -> pdflatex", + "latexmk", + "official-template", + ), + forbidden=ACTIVE_SURFACE_FORBIDDEN, + ), + SurfaceCheck( + path=REPO / "paper/CRPTO_ijds.qmd", + required=( + *COMMON_CLAIM_TOKENS, + "the paper makes three contributions", + "retrospective lockbox replay", + "matched point-pd", + "q_i=(p_i+u_i)/2", + ), + forbidden=ACTIVE_SURFACE_FORBIDDEN, + ), + SurfaceCheck( + path=REPO / "paper/submission/CRPTO_ijds_submission.tex", + required=( + *COMMON_CLAIM_TOKENS, + "the paper makes three contributions", + "retrospective lockbox replay", + "matched point-pd", + ), + forbidden=ACTIVE_SURFACE_FORBIDDEN, + ), + SurfaceCheck( + path=REPO / "paper/supplement_ijds.qmd", + required=( + *COMMON_CLAIM_TOKENS, + "a35. exact alpha replay", + "a36. calibration policy selector", + "a40. matched decision audit", + "retrospective lockbox replay", + ), + forbidden=ACTIVE_SURFACE_FORBIDDEN, + ), + SurfaceCheck( + path=REPO / "paper/submission/CLAIM_AUDIT_MATRIX.md", + required=( + "calibration-selected midpoint", + "a40", + "8.678%", + "7.9025", + ), + forbidden=ACTIVE_SURFACE_FORBIDDEN, + ), + SurfaceCheck( + path=REPO / "docs/research/active_claims_2026-07-04.md", + required=( + *COMMON_CLAIM_TOKENS, + "nine round-number candidates", + "retrospective lockbox replay", + "retired headline claims", + ), + forbidden=("crpto v2", "markov cap", "+27.03%"), + ), + SurfaceCheck( + path=REPO / "configs/crpto_publication_targets.yaml", + required=( + "exact 90% conformal replay", + "q=(p+u)/2", + "outside the submitted claim", + "not acceptance criteria", + ), + forbidden=("crpto v2",), + ), +) + + +def _normalize(text: str) -> str: + """Normalize Markdown and LaTeX enough for robust token checks.""" + lowered = text.lower() + replacements = { + r"\$": "$", + "{,}": ",", + r"\_": "_", + r"\mathrm": "", + r"\gamma": "gamma", + r"\alpha": "alpha", + "\\": "", + "{": "", + "}": "", + "`": "", + ",": "", + } + for old, new in replacements.items(): + lowered = lowered.replace(old, new) + lowered = lowered.replace("\u2013", "-").replace("\u2014", "-") + return re.sub(r"\s+", " ", lowered) + + +def _read_normalized(path: Path) -> str: + return _normalize(path.read_text(encoding="utf-8")) + + +def check_publication_integrity() -> list[str]: + """Return active-manuscript integrity failures.""" + failures: list[str] = [] + for surface in SURFACES: + if not surface.path.is_file(): + failures.append(f"{surface.path.relative_to(REPO)} is missing") + continue + text = _read_normalized(surface.path) + missing = [token for token in surface.required if token not in text] + present_forbidden = [token for token in surface.forbidden if token in text] + rel = surface.path.relative_to(REPO) + failures.extend(f"{rel}: missing required token '{token}'" for token in missing) + failures.extend( + f"{rel}: forbidden token still present '{token}'" for token in present_forbidden + ) + return failures + + +def main() -> int: + """CLI entry point.""" + failures = check_publication_integrity() + if failures: + for failure in failures: + logger.error(failure) + return 1 + logger.success("Active IJDS publication surfaces are claim-synchronized.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/compile_ijds_submission.py b/scripts/compile_ijds_submission.py new file mode 100644 index 0000000..9cd7eba --- /dev/null +++ b/scripts/compile_ijds_submission.py @@ -0,0 +1,195 @@ +"""Compile and sanity-check the official IJDS LaTeX submission PDF.""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +from loguru import logger + +ROOT = Path(__file__).resolve().parents[1] +SUBMISSION_DIR = ROOT / "paper" / "submission" +REPORT_DIR = ROOT / "reports" / "ci" +TEX_NAME = "CRPTO_ijds_submission.tex" +JOB_NAME = "CRPTO_ijds_submission" + + +@dataclass(frozen=True) +class LatexScan: + """Summary of a compiled LaTeX submission surface.""" + + pages: int | None + blg_warnings: tuple[str, ...] + log_failures: tuple[str, ...] + + @property + def ok(self) -> bool: + return not self.blg_warnings and not self.log_failures + + +def _run(command: list[str], *, cwd: Path, env: dict[str, str], transcript: Path) -> int: + logger.info("Running: {}", " ".join(command)) + completed = subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + ) + transcript.parent.mkdir(parents=True, exist_ok=True) + with transcript.open("a", encoding="utf-8", errors="replace") as handle: + handle.write(f"\n$ {' '.join(command)}\n") + handle.write(completed.stdout) + return int(completed.returncode) + + +def _submission_env() -> dict[str, str]: + env = os.environ.copy() + if not env.get("WINDIR") and env.get("SystemRoot"): + env["WINDIR"] = env["SystemRoot"] + return env + + +def _windows_latexmk_script(latexmk_executable: str | Path) -> Path | None: + """Locate ``latexmk.pl`` beside a TeX Live/TinyTeX Windows wrapper.""" + executable = Path(latexmk_executable).resolve() + for root in executable.parents: + script = root / "texmf-dist" / "scripts" / "latexmk" / "latexmk.pl" + if script.is_file(): + return script + return None + + +def _latexmk_command() -> list[str] | None: + """Return a working latexmk launcher, bypassing fragile TinyTeX wrappers.""" + executable = shutil.which("latexmk") + if executable is None: + return None + if os.name == "nt" and (perl := shutil.which("perl")) is not None: + script = _windows_latexmk_script(executable) + if script is not None: + return [perl, str(script)] + return [executable] + + +def _manual_pdflatex_bibtex(cwd: Path, env: dict[str, str], transcript: Path) -> int: + sequence = [ + ["pdflatex", "-interaction=nonstopmode", "-halt-on-error", TEX_NAME], + ["bibtex", JOB_NAME], + ["pdflatex", "-interaction=nonstopmode", "-halt-on-error", TEX_NAME], + ["pdflatex", "-interaction=nonstopmode", "-halt-on-error", TEX_NAME], + ] + for command in sequence: + code = _run(command, cwd=cwd, env=env, transcript=transcript) + if code != 0: + return code + return 0 + + +def compile_submission(*, prefer_manual: bool = False) -> int: + """Compile the official submission with latexmk, falling back to manual passes.""" + if not (SUBMISSION_DIR / TEX_NAME).is_file(): + logger.error("Missing {}", (SUBMISSION_DIR / TEX_NAME).relative_to(ROOT)) + return 2 + + env = _submission_env() + transcript = REPORT_DIR / "ijds-latex-build.txt" + if transcript.exists(): + transcript.unlink() + latexmk_command = _latexmk_command() + if prefer_manual or latexmk_command is None: + if latexmk_command is None: + logger.warning("latexmk unavailable; using manual pdflatex/BibTeX fallback.") + return _manual_pdflatex_bibtex(SUBMISSION_DIR, env, transcript) + + latexmk_code = _run( + [*latexmk_command, "-pdf", "-gg", "-interaction=nonstopmode", TEX_NAME], + cwd=SUBMISSION_DIR, + env=env, + transcript=transcript, + ) + if latexmk_code == 0: + logger.info("LaTeX transcript: {}", transcript.relative_to(ROOT)) + return 0 + + logger.warning("latexmk failed with code {}; using manual fallback.", latexmk_code) + code = _manual_pdflatex_bibtex(SUBMISSION_DIR, env, transcript) + logger.info("LaTeX transcript: {}", transcript.relative_to(ROOT)) + return code + + +def scan_submission_logs() -> LatexScan: + """Inspect `.blg` and `.log` outputs for bibliography/reference drift.""" + blg_path = SUBMISSION_DIR / f"{JOB_NAME}.blg" + log_path = SUBMISSION_DIR / f"{JOB_NAME}.log" + blg_text = blg_path.read_text(encoding="utf-8", errors="replace") if blg_path.exists() else "" + log_text = log_path.read_text(encoding="utf-8", errors="replace") if log_path.exists() else "" + + blg_warnings = tuple( + line.strip() for line in blg_text.splitlines() if line.strip().startswith("Warning--") + ) + + checks = { + "undefined references": r"There were undefined references", + "undefined citations": r"Citation `[^`]+`.*undefined", + "undefined labels": r"Reference `[^`]+`.*undefined", + "rerun requested": r"Rerun to get cross-references right|Label\(s\) may have changed", + } + log_failures = tuple(name for name, pattern in checks.items() if re.search(pattern, log_text)) + + pages: int | None = None + matches = re.findall(r"Output written on .+?\((\d+) pages?,", log_text) + if matches: + pages = int(matches[-1]) + + return LatexScan(pages=pages, blg_warnings=blg_warnings, log_failures=log_failures) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--manual", + action="store_true", + help="Skip latexmk and use pdflatex + bibtex + pdflatex + pdflatex directly.", + ) + parser.add_argument( + "--scan-only", + action="store_true", + help="Do not compile; only inspect existing .log and .blg files.", + ) + args = parser.parse_args(argv) + + if not args.scan_only: + code = compile_submission(prefer_manual=args.manual) + if code != 0: + logger.error("Official IJDS LaTeX compile failed with code {}.", code) + return code + + scan = scan_submission_logs() + if scan.blg_warnings: + logger.error("BibTeX warnings:\n{}", "\n".join(scan.blg_warnings)) + if scan.log_failures: + logger.error("LaTeX convergence failures: {}", ", ".join(scan.log_failures)) + if scan.pages is None: + logger.warning("Could not read page count from the LaTeX log.") + else: + logger.success("Official IJDS PDF page count: {}", scan.pages) + + if not scan.ok: + return 1 + logger.success("Official IJDS LaTeX build is citation/reference clean.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/experiments/ijds_policy_support.py b/scripts/experiments/ijds_policy_support.py new file mode 100644 index 0000000..10ab494 --- /dev/null +++ b/scripts/experiments/ijds_policy_support.py @@ -0,0 +1,188 @@ +"""Shared exact-alpha panel loading and policy evaluation for IJDS experiments.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from src.models.conformal_alpha_grid import alpha_interval_columns # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + compute_funded_certificate_metrics, +) +from src.optimization.input_alignment import align_candidate_intervals # noqa: E402 +from src.optimization.policy_evaluation import ( # noqa: E402 + PolicyAllocationResult, + solve_policy_allocation, +) +from src.optimization.policy_selection import ( # noqa: E402 + LinearPolicyCandidate, + temporal_period_labels, +) +from src.utils.script_helpers import parse_percent_series, resolve_repo_artifact_path # noqa: E402 + + +def load_policy_panel(config: dict[str, Any], *, root: Path = ROOT) -> pd.DataFrame: + """Load candidates and exact-alpha intervals under a strict ID contract.""" + source = config["source"] + design = config["design"] + execution = config["execution"] + candidate_value = str(source.get("candidate_path", "data/processed/test_fe.parquet")) + candidate_path = resolve_repo_artifact_path(candidate_value, root=root) + exact_grid_value = str(source.get("exact_alpha_grid_path", "")).strip() + if not exact_grid_value: + raise ValueError("Active IJDS policy evaluation requires exact_alpha_grid_path.") + exact_grid_path = resolve_repo_artifact_path(exact_grid_value, root=root) + exact_alignment = align_candidate_intervals( + pd.read_parquet(candidate_path), + pd.read_parquet(exact_grid_path), + max_candidates=0, + random_state=int(execution.get("random_seed", 42)), + ) + if exact_alignment.mode != "id": + raise RuntimeError( + "Active IJDS policy evaluation requires one-to-one candidate/exact-grid ID alignment." + ) + panel = exact_alignment.candidates.copy() + exact = exact_alignment.intervals + low_column, high_column = alpha_interval_columns(float(design["alpha"])) + required = {"y_pred", low_column, high_column} + missing = sorted(required.difference(exact.columns)) + if missing: + raise KeyError(f"Exact alpha grid is missing columns: {missing}") + + panel["_pd_point"] = exact["y_pred"].to_numpy(dtype=float) + panel["_pd_low"] = exact[low_column].to_numpy(dtype=float) + panel["_pd_high"] = exact[high_column].to_numpy(dtype=float) + exact_outcome = pd.to_numeric(exact["y_true"], errors="raise").to_numpy(dtype=float) + if "default_flag" in panel.columns: + candidate_outcome = pd.to_numeric(panel["default_flag"], errors="raise").to_numpy( + dtype=float + ) + if not np.array_equal(candidate_outcome, exact_outcome): + raise ValueError("Candidate default_flag does not match exact-grid y_true by ID.") + panel["_outcome"] = exact_outcome + panel["_loan_amount"] = pd.to_numeric(panel["loan_amnt"], errors="coerce").fillna(1.0) + panel["_int_rate"] = parse_percent_series(panel["int_rate"]) + panel["_period"] = temporal_period_labels( + panel["issue_d"], + combine_years_from=int(design["combine_years_from"]), + ) + panel.attrs.update( + { + "candidate_path": str(candidate_path), + "exact_alpha_grid_path": str(exact_grid_path), + "alignment_mode": exact_alignment.mode, + } + ) + return panel + + +def solve_candidate( + frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + *, + config: dict[str, Any], + robust: bool = True, +) -> PolicyAllocationResult: + """Solve one declared policy on one aligned panel.""" + design = config["design"] + execution = config["execution"] + return solve_policy_allocation( + loans=frame, + pd_point=frame["_pd_point"].to_numpy(dtype=float), + pd_low=frame["_pd_low"].to_numpy(dtype=float), + pd_high=frame["_pd_high"].to_numpy(dtype=float), + lgd=np.full(len(frame), float(design["lgd"]), dtype=float), + int_rates=frame["_int_rate"].to_numpy(dtype=float), + total_budget=float(design["budget"]), + max_concentration=float(design["max_concentration"]), + risk_tolerance=float(candidate.risk_tolerance), + robust=robust, + uncertainty_aversion=float(candidate.uncertainty_aversion) if robust else 0.0, + min_budget_utilization=float(candidate.min_budget_utilization), + pd_cap_slack_penalty=float(candidate.pd_cap_slack_penalty), + policy_mode=candidate.policy_mode, + gamma=float(candidate.gamma), + delta_cap_quantile=float(candidate.delta_cap_quantile), + tail_focus_quantile=float(candidate.tail_focus_quantile), + time_limit=int(execution["time_limit"]), + threads=int(execution["threads"]), + solver_backend=str(execution["solver_backend"]), + random_seed=int(execution.get("random_seed", 42)), + ) + + +def evaluate_candidate( + frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + *, + config: dict[str, Any], + robust: bool, + period: str, +) -> tuple[dict[str, Any], PolicyAllocationResult]: + """Solve and score one policy on one evaluation period.""" + result = solve_candidate(frame, candidate, config=config, robust=robust) + exposure = result.allocation * frame["_loan_amount"].to_numpy(dtype=float) + total_allocated = float(exposure.sum()) + if total_allocated <= 0.0: + raise RuntimeError(f"Policy {candidate.candidate_id} allocated no capital in {period}.") + weights = exposure / total_allocated + outcomes = frame["_outcome"].to_numpy(dtype=float) + alpha = float(config["design"]["alpha"]) + certificate = compute_funded_certificate_metrics( + weights, + outcomes=outcomes, + pd_point=frame["_pd_point"].to_numpy(dtype=float), + pd_high=frame["_pd_high"].to_numpy(dtype=float), + pd_effective=result.effective_pd, + alpha=alpha, + risk_tolerance=float(candidate.risk_tolerance), + pd_cap_slack=float(result.solution.get("pd_cap_slack", 0.0)), + ) + funded = result.allocation > 0.01 + rates = frame["_int_rate"].to_numpy(dtype=float) + lgd = float(config["design"]["lgd"]) + realized_return = float( + np.sum( + np.where( + funded & (outcomes.astype(int) == 1), + -lgd * exposure, + np.where(funded, rates * exposure, 0.0), + ) + ) + ) + record: dict[str, Any] = { + "period": period, + **candidate.to_record(), + "solver_status": str(result.solution.get("solver_status", "unknown")), + "objective_risk_mode": result.objective_risk_mode, + "expected_objective": float(result.solution.get("objective_value", float("nan"))), + "n_panel": int(len(frame)), + "n_funded": int(certificate.n_funded), + "total_allocated": total_allocated, + "realized_return": realized_return, + "weighted_outcome": certificate.weighted_outcome, + "weighted_miscoverage": certificate.weighted_miscoverage, + "weighted_pd_point": certificate.weighted_pd_point, + "weighted_pd_effective": certificate.weighted_pd_effective, + "gamma_cp": certificate.gamma_cp, + "gamma_internalized": certificate.gamma_internalized, + "gamma_residual": certificate.gamma_residual, + "endpoint_budget": certificate.endpoint_budget, + "markov_loss_threshold": certificate.markov_loss_threshold, + "realized_risk_tolerance_excess": certificate.realized_risk_tolerance_excess, + "screen_V_leq_sqrt_alpha": bool( + certificate.weighted_miscoverage <= certificate.sqrt_alpha + 1e-12 + ), + "screen_risk_excess_leq_alpha": bool( + certificate.realized_risk_tolerance_excess <= alpha + 1e-12 + ), + } + return record, result diff --git a/scripts/experiments/run_champion_claim_max_downstream.py b/scripts/experiments/run_champion_claim_max_downstream.py index db217c3..4c90501 100644 --- a/scripts/experiments/run_champion_claim_max_downstream.py +++ b/scripts/experiments/run_champion_claim_max_downstream.py @@ -26,6 +26,31 @@ CHAMPION_BRIER = 0.15439302183275685 CHAMPION_ECE = 0.006998009158194006 +CUOPT_FLAG_MAP = { + "presolve": "--cuopt-presolve", + "method": "--cuopt-method", + "pdlp_solver_mode": "--cuopt-pdlp-solver-mode", + "pdlp_precision": "--cuopt-pdlp-precision", + "crossover": "--cuopt-crossover", + "first_primal_feasible": "--cuopt-first-primal-feasible", + "save_best_primal_solution": "--cuopt-save-best-primal-solution", + "infeasibility_detection": "--cuopt-infeasibility-detection", + "strict_infeasibility": "--cuopt-strict-infeasibility", + "per_constraint_residual": "--cuopt-per-constraint-residual", + "dual_postsolve": "--cuopt-dual-postsolve", + "dualize": "--cuopt-dualize", + "folding": "--cuopt-folding", + "augmented": "--cuopt-augmented", + "ordering": "--cuopt-ordering", + "cudss_deterministic": "--cuopt-cudss-deterministic", + "eliminate_dense_columns": "--cuopt-eliminate-dense-columns", + "iteration_limit": "--cuopt-iteration-limit", + "num_cpu_threads": "--cuopt-num-cpu-threads", + "num_gpus": "--cuopt-num-gpus", + "log_to_console": "--cuopt-log-to-console", + "log_dir": "--cuopt-log-dir", +} + def _utc_now() -> str: return datetime.now(tz=UTC).isoformat() @@ -259,22 +284,35 @@ def _conformal_intervals_for_status(status: dict[str, Any]) -> Path: ) -def _portfolio_command( +def _profile_section(portfolio_profile: dict[str, Any], key: str) -> dict[str, Any]: + return dict(portfolio_profile.get(key, {}) or {}) + + +def _append_option(command: list[str], flag: str, value: Any) -> None: + if value is not None and str(value).strip(): + command.extend([flag, str(value)]) + + +def _append_int_option(command: list[str], flag: str, value: Any) -> None: + if value is not None and str(value).strip(): + command.extend([flag, str(int(value))]) + + +def _portfolio_base_command( *, portfolio_profile: dict[str, Any], conformal_intervals_path: Path, run_label: str, output_dir: Path, model_dir: Path, + grids: dict[str, Any], + frontier: dict[str, Any], + incumbent: dict[str, Any], + execution: dict[str, Any], ) -> list[str]: - grids = dict(portfolio_profile.get("grids", {}) or {}) - frontier = dict(portfolio_profile.get("frontier", {}) or {}) - incumbent = dict(portfolio_profile.get("incumbent_region", {}) or {}) - execution = dict(portfolio_profile.get("execution", {}) or {}) - cuopt = dict(portfolio_profile.get("cuopt", {}) or {}) python_executable = str(execution.get("python_executable") or sys.executable) policy_modes = ",".join(str(x) for x in portfolio_profile.get("candidate_policy_families", [])) - command = [ + return [ python_executable, "scripts/search/run_portfolio_bound_aware_search.py", "--config", @@ -328,54 +366,62 @@ def _portfolio_command( "--exact-solver-backend", str(execution.get("exact_solver_backend", "highs")), ] - if frontier.get("exact_max_candidates") is not None: - command.extend(["--exact-max-candidates", str(int(frontier["exact_max_candidates"]))]) + + +def _append_frontier_options( + command: list[str], + *, + grids: dict[str, Any], + frontier: dict[str, Any], +) -> None: + _append_int_option(command, "--exact-max-candidates", frontier.get("exact_max_candidates")) exact_random_states = frontier.get("exact_random_states", grids.get("exact_random_states")) - if exact_random_states is not None and str(exact_random_states).strip(): - command.extend(["--exact-random-states", str(exact_random_states)]) - exact_checkpoint_every = frontier.get("exact_checkpoint_every") - if exact_checkpoint_every is not None and str(exact_checkpoint_every).strip(): - command.extend(["--exact-checkpoint-every", str(int(exact_checkpoint_every))]) - exact_threads = frontier.get("exact_threads") - if exact_threads is not None and str(exact_threads).strip(): - command.extend(["--exact-threads", str(int(exact_threads))]) - budget_profiles = grids.get("budget_profiles") - if budget_profiles is not None and str(budget_profiles).strip(): - command.extend(["--budget-profiles", str(budget_profiles)]) + _append_option(command, "--exact-random-states", exact_random_states) + _append_int_option(command, "--exact-checkpoint-every", frontier.get("exact_checkpoint_every")) + _append_int_option(command, "--exact-threads", frontier.get("exact_threads")) + _append_option(command, "--budget-profiles", grids.get("budget_profiles")) + + +def _append_execution_options(command: list[str], *, execution: dict[str, Any]) -> None: if bool(execution.get("frontier_only", False)): command.append("--frontier-only") - if str(execution.get("exact_python_executable", "")).strip(): - command.extend(["--exact-python-executable", str(execution["exact_python_executable"])]) - cuopt_flag_map = { - "presolve": "--cuopt-presolve", - "method": "--cuopt-method", - "pdlp_solver_mode": "--cuopt-pdlp-solver-mode", - "pdlp_precision": "--cuopt-pdlp-precision", - "crossover": "--cuopt-crossover", - "first_primal_feasible": "--cuopt-first-primal-feasible", - "save_best_primal_solution": "--cuopt-save-best-primal-solution", - "infeasibility_detection": "--cuopt-infeasibility-detection", - "strict_infeasibility": "--cuopt-strict-infeasibility", - "per_constraint_residual": "--cuopt-per-constraint-residual", - "dual_postsolve": "--cuopt-dual-postsolve", - "dualize": "--cuopt-dualize", - "folding": "--cuopt-folding", - "augmented": "--cuopt-augmented", - "ordering": "--cuopt-ordering", - "cudss_deterministic": "--cuopt-cudss-deterministic", - "eliminate_dense_columns": "--cuopt-eliminate-dense-columns", - "iteration_limit": "--cuopt-iteration-limit", - "num_cpu_threads": "--cuopt-num-cpu-threads", - "num_gpus": "--cuopt-num-gpus", - "log_to_console": "--cuopt-log-to-console", - "log_dir": "--cuopt-log-dir", - } - for key, flag in cuopt_flag_map.items(): - value = cuopt.get(key) - if value is not None and str(value).strip() != "": - command.extend([flag, str(value)]) + _append_option(command, "--exact-python-executable", execution.get("exact_python_executable")) + + +def _append_cuopt_options(command: list[str], *, cuopt: dict[str, Any]) -> None: + for key, flag in CUOPT_FLAG_MAP.items(): + _append_option(command, flag, cuopt.get(key)) for key, value in dict(cuopt.get("extra_parameters", {}) or {}).items(): command.extend(["--cuopt-extra-parameter", f"{key}={value}"]) + + +def _portfolio_command( + *, + portfolio_profile: dict[str, Any], + conformal_intervals_path: Path, + run_label: str, + output_dir: Path, + model_dir: Path, +) -> list[str]: + grids = _profile_section(portfolio_profile, "grids") + frontier = _profile_section(portfolio_profile, "frontier") + incumbent = _profile_section(portfolio_profile, "incumbent_region") + execution = _profile_section(portfolio_profile, "execution") + cuopt = _profile_section(portfolio_profile, "cuopt") + command = _portfolio_base_command( + portfolio_profile=portfolio_profile, + conformal_intervals_path=conformal_intervals_path, + run_label=run_label, + output_dir=output_dir, + model_dir=model_dir, + grids=grids, + frontier=frontier, + incumbent=incumbent, + execution=execution, + ) + _append_frontier_options(command, grids=grids, frontier=frontier) + _append_execution_options(command, execution=execution) + _append_cuopt_options(command, cuopt=cuopt) return command diff --git a/scripts/experiments/run_champion_reopen_hpo.py b/scripts/experiments/run_champion_reopen_hpo.py index 89c25d4..efdb5cc 100644 --- a/scripts/experiments/run_champion_reopen_hpo.py +++ b/scripts/experiments/run_champion_reopen_hpo.py @@ -219,9 +219,10 @@ def main() -> None: pool_features, feature_config=feature_config, ) - core_features = _resolve_core_features(feature_config, train.columns) - catboost_features = _resolve_catboost_features(feature_config, train.columns) - woe_features = _resolve_woe_features(feature_config, train.columns) + train_columns = [str(column) for column in train.columns] + core_features = _resolve_core_features(feature_config, train_columns) + catboost_features = _resolve_catboost_features(feature_config, train_columns) + woe_features = _resolve_woe_features(feature_config, train_columns) train_fit, train_val = temporal_train_val_split( train, diff --git a/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py b/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py new file mode 100644 index 0000000..21f86e7 --- /dev/null +++ b/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py @@ -0,0 +1,597 @@ +"""Select a simple CRPTO policy without consulting OOT outcomes.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pickle +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import yaml +from loguru import logger + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from scripts.experiments.ijds_policy_support import ( # noqa: E402 + evaluate_candidate, + load_policy_panel, + solve_candidate, +) +from scripts.generate_conformal_intervals import ( # noqa: E402 + _build_probability_lookups, + _build_tuning_split, + _load_conformal_inputs, +) +from src.models.conformal_alpha_grid import ( # noqa: E402 + FrozenConformalRecipe, + compute_exact_alpha_intervals, +) +from src.optimization.policy_evaluation import PolicyAllocationResult # noqa: E402 +from src.optimization.policy_selection import ( # noqa: E402 + FORBIDDEN_POLICY_SELECTION_COLUMNS, + LinearPolicyCandidate, + build_linear_policy_grid, + endpoint_cap_stability, + select_policy_result_ex_ante, +) +from src.utils.script_helpers import ( # noqa: E402 + parse_percent_series, + resolve_repo_artifact_path, + write_json, +) + +DEFAULT_CONFIG = ( + ROOT / "configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml" +) +FORBIDDEN_SELECTOR_COLUMNS = FORBIDDEN_POLICY_SELECTION_COLUMNS +SELECTOR_INPUT_COLUMNS = ( + "id", + "loan_amnt", + "purpose", + "grade", + "term", + "verification_status", + "issue_d", + "_pd_point", + "_pd_low", + "_pd_high", + "_loan_amount", + "_int_rate", +) + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Experiment config must contain a mapping.") + return payload + + +def _load_pickle_mapping(path: Path) -> dict[str, Any]: + with path.open("rb") as handle: + payload = pickle.load(handle) + if not isinstance(payload, dict): + raise TypeError("Frozen conformal results must contain a mapping.") + return payload + + +def _experiment_paths(run_tag: str) -> tuple[Path, Path]: + data_dir = ROOT / "data/processed/experiments/champion_reopen" / run_tag / "portfolio" + model_dir = ROOT / "models/experiments/champion_reopen" / run_tag / "portfolio" + data_dir.mkdir(parents=True, exist_ok=True) + model_dir.mkdir(parents=True, exist_ok=True) + return data_dir, model_dir + + +def _endpoint_budget_cap(config: dict[str, Any]) -> float: + design = config["design"] + if "endpoint_budget_cap" in design: + return float(design["endpoint_budget_cap"]) + if "markov_threshold_cap" in design: + return float(design["markov_threshold_cap"]) - float(np.sqrt(design["alpha"])) + raise KeyError("Experiment design requires endpoint_budget_cap.") + + +def _selector_input_frame(frame: pd.DataFrame) -> pd.DataFrame: + """Expose only columns required to solve the outcome-free policy grid.""" + required = {"loan_amnt", "purpose", "_pd_point", "_pd_low", "_pd_high", "_int_rate"} + missing = sorted(required.difference(frame.columns)) + if missing: + raise KeyError(f"Calibration selector input is missing columns: {missing}") + return frame.loc[:, [column for column in SELECTOR_INPUT_COLUMNS if column in frame]].copy() + + +def _load_calibration_selection_panel( + config: dict[str, Any], +) -> tuple[pd.DataFrame, pd.Series, FrozenConformalRecipe, dict[str, Any]]: + source = config["source"] + design = config["design"] + os.environ["UPSTREAM_CANONICAL_RUN_TAG"] = str(source["upstream_canonical_run_tag"]) + results_path = resolve_repo_artifact_path(source["conformal_results_path"], root=ROOT) + payload = _load_pickle_mapping(results_path) + recipe = FrozenConformalRecipe.from_results_payload(payload) + inputs = _load_conformal_inputs( + calibration_fraction=recipe.calibration_fraction, + calibrator_override_path=(str(payload.get("calibrator_override_path", "")).strip() or None), + ) + split = _build_tuning_split( + cal_df=inputs.cal_df, + test_df=inputs.test_df, + X_cal=inputs.X_cal, + y_cal=inputs.y_cal, + group_cal_base=inputs.group_cal_base, + y_prob_cal_raw=inputs.y_prob_cal_raw, + tuning_holdout_ratio=recipe.tuning_holdout_ratio, + tuning_random_state=recipe.tuning_random_state, + ) + probability_fit, probability_tune, _probability_test = _build_probability_lookups( + inputs, + split, + ) + intervals = compute_exact_alpha_intervals( + recipe=recipe, + target_alpha=float(design["alpha"]), + y_cal=split.y_cal_fit, + interval_probability_cal=probability_fit["calibrated"], + interval_probability_eval=probability_tune["calibrated"], + partition_probability_cal=probability_fit[recipe.partition_probability_source], + partition_probability_eval=probability_tune[recipe.partition_probability_source], + base_groups_cal=split.group_cal_fit_base, + base_groups_eval=split.group_tune_base, + issue_dates_eval=split.issue_tune, + ) + panel = inputs.cal_df.iloc[split.idx_cal_tune].reset_index(drop=True).copy() + panel["_pd_point"] = intervals.point + panel["_pd_low"] = intervals.low + panel["_pd_high"] = intervals.high + panel["_loan_amount"] = pd.to_numeric(panel["loan_amnt"], errors="coerce").fillna(1.0) + panel["_int_rate"] = parse_percent_series(panel["int_rate"]) + outcomes = pd.Series(split.y_tune.to_numpy(dtype=float), name="_outcome") + months = pd.to_datetime(panel["issue_d"], errors="raise").dt.to_period("M").astype(str) + panel = _selector_input_frame(panel) + panel["_month"] = months.to_numpy(dtype=str) + metadata = { + "conformal_results_path": str(results_path.relative_to(ROOT)), + "calibration_fit_rows": int(len(split.idx_cal_fit)), + "calibration_holdout_rows": int(len(split.idx_cal_tune)), + "calibration_holdout_start": str(pd.to_datetime(split.issue_tune).min().date()), + "calibration_holdout_end": str(pd.to_datetime(split.issue_tune).max().date()), + "target_alpha": intervals.target_alpha, + "used_alpha": intervals.used_alpha, + "partition": recipe.partition, + } + return panel, outcomes, recipe, metadata + + +def _measure_ex_ante_solution( + frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + result: PolicyAllocationResult, + *, + alpha: float, +) -> dict[str, Any]: + exposure = result.allocation * frame["_loan_amount"].to_numpy(dtype=float) + total_allocated = float(exposure.sum()) + if total_allocated <= 0.0: + raise RuntimeError(f"Policy {candidate.candidate_id} allocated no calibration capital.") + weights = exposure / total_allocated + point = frame["_pd_point"].to_numpy(dtype=float) + high = frame["_pd_high"].to_numpy(dtype=float) + weighted_point = float(np.sum(weights * point)) + weighted_effective = float(np.sum(weights * result.effective_pd)) + endpoint_budget = float(np.sum(weights * high)) + return { + **candidate.to_record(), + "solver_status": str(result.solution.get("solver_status", "unknown")), + "objective_risk_mode": result.objective_risk_mode, + "expected_objective": float(result.solution["objective_value"]), + "n_panel": int(len(frame)), + "n_funded": int(np.count_nonzero(result.allocation > 0.01)), + "total_allocated": total_allocated, + "weighted_pd_point": weighted_point, + "weighted_pd_effective": weighted_effective, + "gamma_cp": float(np.sum(weights * (high - point))), + "gamma_internalized": float(np.sum(weights * (result.effective_pd - point))), + "gamma_residual": float(np.sum(weights * (high - result.effective_pd))), + "endpoint_budget": endpoint_budget, + "markov_loss_threshold": endpoint_budget + float(np.sqrt(alpha)), + "effective_pd_cap_slack": float(candidate.risk_tolerance - weighted_effective), + } + + +def _run_calibration_grid( + panel: pd.DataFrame, + candidates: list[LinearPolicyCandidate], + *, + config: dict[str, Any], + output_path: Path, +) -> pd.DataFrame: + selector_panel = _selector_input_frame(panel) + rows: list[dict[str, Any]] = [] + for candidate in candidates: + logger.info("Calibration selector evaluating {}", candidate.candidate_id) + result = solve_candidate(selector_panel, candidate, config=config) + rows.append( + _measure_ex_ante_solution( + selector_panel, + candidate, + result, + alpha=float(config["design"]["alpha"]), + ) + ) + output = pd.DataFrame(rows) + if FORBIDDEN_SELECTOR_COLUMNS.intersection(output.columns): + raise AssertionError("Calibration selector artifact contains outcome-derived columns.") + output.to_parquet(output_path, index=False) + return output + + +def _match_candidate( + candidates: list[LinearPolicyCandidate], + settings: dict[str, Any], +) -> LinearPolicyCandidate: + matches = [ + candidate + for candidate in candidates + if np.isclose(candidate.risk_tolerance, float(settings["risk_tolerance"])) + and np.isclose(candidate.gamma, float(settings["gamma"])) + and np.isclose(candidate.uncertainty_aversion, float(settings["uncertainty_aversion"])) + ] + if len(matches) != 1: + raise ValueError(f"Policy settings must match exactly one candidate, got {len(matches)}.") + return matches[0] + + +def _comparison_policies( + selected: LinearPolicyCandidate, + incumbent: LinearPolicyCandidate, +) -> tuple[tuple[str, LinearPolicyCandidate, bool], ...]: + point = LinearPolicyCandidate( + candidate_id="point-pd", + risk_tolerance=selected.risk_tolerance, + gamma=0.0, + uncertainty_aversion=0.0, + policy_mode="point_estimate", + ) + return ( + ("calibration_selected", selected, True), + ("incumbent_linear", incumbent, True), + ("point_pd_matched_tau", point, False), + ) + + +def _evaluate_fixed_policies( + panel: pd.DataFrame, + selected: LinearPolicyCandidate, + incumbent: LinearPolicyCandidate, + *, + config: dict[str, Any], +) -> tuple[pd.DataFrame, pd.DataFrame]: + periods = ["full_oot", *list(config["design"]["period_order"])] + rows: list[dict[str, Any]] = [] + allocation_frames: list[pd.DataFrame] = [] + for period in periods: + frame = ( + panel.reset_index(drop=True) + if period == "full_oot" + else panel.loc[panel["_period"].astype(str).eq(period)].reset_index(drop=True) + ) + for role, candidate, robust in _comparison_policies(selected, incumbent): + record, result = evaluate_candidate( + frame, + candidate, + config=config, + robust=robust, + period=period, + ) + rows.append({"role": role, **record}) + if period == "full_oot": + allocation_frames.append( + _funded_allocation_frame( + frame, + result, + role=role, + lgd=float(config["design"]["lgd"]), + ) + ) + return pd.DataFrame(rows), pd.concat(allocation_frames, ignore_index=True) + + +def _evaluate_calibration_audit( + panel: pd.DataFrame, + selected: LinearPolicyCandidate, + incumbent: LinearPolicyCandidate, + *, + config: dict[str, Any], + period: str, +) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + frame = panel.reset_index(drop=True) + for role, candidate, robust in _comparison_policies(selected, incumbent): + record, _result = evaluate_candidate( + frame, + candidate, + config=config, + robust=robust, + period=period, + ) + rows.append({"role": role, **record}) + return pd.DataFrame(rows) + + +def _funded_allocation_frame( + frame: pd.DataFrame, + result: PolicyAllocationResult, + *, + role: str, + lgd: float, +) -> pd.DataFrame: + allocation = result.allocation + funded = allocation > 0.01 + selected = frame.loc[funded].reset_index(drop=True).copy() + selected_allocation = allocation[funded] + exposure = selected_allocation * selected["_loan_amount"].to_numpy(dtype=float) + total_exposure = float(exposure.sum()) + outcome = selected["_outcome"].to_numpy(dtype=float) + rates = selected["_int_rate"].to_numpy(dtype=float) + point = selected["_pd_point"].to_numpy(dtype=float) + high = selected["_pd_high"].to_numpy(dtype=float) + effective = result.effective_pd[funded] + if "sub_grade" in selected.columns: + loan_grade = selected["sub_grade"].astype(str).str[:1] + elif "int_rate_bucket__grade" in selected.columns: + loan_grade = selected["int_rate_bucket__grade"].astype(str).str.rsplit("__").str[-1] + else: + loan_grade = selected["grade"].astype(str) + output = pd.DataFrame( + { + "role": role, + "id": selected["id"].astype(str), + "issue_d": selected["issue_d"], + "grade": loan_grade, + "conformal_group": selected["grade"].astype(str), + "loan_amnt": selected["_loan_amount"].to_numpy(dtype=float), + "int_rate": rates, + "outcome": outcome, + "pd_point": point, + "pd_low": selected["_pd_low"].to_numpy(dtype=float), + "pd_high": high, + "pd_effective": effective, + "allocation": selected_allocation, + "funded_exposure": exposure, + "funded_weight": exposure / total_exposure, + "miscoverage": (outcome > high).astype(int), + "expected_return_contribution": exposure * (rates - point * float(lgd)), + "realized_return_contribution": np.where( + outcome.astype(int) == 1, + -float(lgd) * exposure, + rates * exposure, + ), + } + ) + return output + + +def _contrast_payload(evaluation: pd.DataFrame) -> dict[str, Any]: + output: dict[str, Any] = {} + for period in ("full_oot", "2020+"): + period_rows = evaluation.loc[evaluation["period"].eq(period)].set_index("role") + selected = period_rows.loc["calibration_selected"] + point = period_rows.loc["point_pd_matched_tau"] + incumbent = period_rows.loc["incumbent_linear"] + output[period] = { + "selected_realized_return": float(selected["realized_return"]), + "selected_weighted_outcome": float(selected["weighted_outcome"]), + "selected_markov_threshold": float(selected["markov_loss_threshold"]), + "return_cost_vs_point": float(point["realized_return"] - selected["realized_return"]), + "default_delta_vs_point": float( + selected["weighted_outcome"] - point["weighted_outcome"] + ), + "threshold_delta_vs_point": float( + selected["markov_loss_threshold"] - point["markov_loss_threshold"] + ), + "return_delta_vs_incumbent": float( + selected["realized_return"] - incumbent["realized_return"] + ), + "default_delta_vs_incumbent": float( + selected["weighted_outcome"] - incumbent["weighted_outcome"] + ), + } + return output + + +def _git_commit() -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def run(config_path: Path) -> dict[str, Any]: + config = _load_config(config_path) + run_tag = str(config["run_tag"]) + data_dir, model_dir = _experiment_paths(run_tag) + calibration_holdout, calibration_outcomes, recipe, calibration_metadata = ( + _load_calibration_selection_panel(config) + ) + design = config["design"] + selection_period = str(design.get("selection_period", "")).strip() + audit_period = str(design.get("audit_period", "")).strip() + selection_mask = ( + calibration_holdout["_month"].eq(selection_period) + if selection_period + else pd.Series(True, index=calibration_holdout.index) + ) + selection_panel = calibration_holdout.loc[selection_mask].reset_index(drop=True).copy() + if selection_panel.empty: + raise ValueError(f"Calibration selection period has no rows: {selection_period!r}") + audit_mask = ( + calibration_holdout["_month"].eq(audit_period) + if audit_period + else pd.Series(False, index=calibration_holdout.index) + ) + audit_panel = calibration_holdout.loc[audit_mask].reset_index(drop=True).copy() + if audit_period and audit_panel.empty: + raise ValueError(f"Calibration audit period has no rows: {audit_period!r}") + audit_outcomes = calibration_outcomes.loc[audit_mask.to_numpy()].reset_index(drop=True) + calibration_metadata.update( + { + "selection_period": selection_period or "full_calibration_holdout", + "selection_rows": int(len(selection_panel)), + "audit_period": audit_period or None, + "audit_rows": int(len(audit_panel)), + "outcomes_isolated_until_post_selection_audit": True, + } + ) + grid_config = config["policy_grid"] + candidates = build_linear_policy_grid( + risk_tolerances=[float(value) for value in grid_config["risk_tolerances"]], + gammas=[float(value) for value in grid_config["gammas"]], + uncertainty_aversions=[float(value) for value in grid_config["uncertainty_aversions"]], + ) + selection_results = _run_calibration_grid( + selection_panel, + candidates, + config=config, + output_path=data_dir / "calibration_policy_selection_grid.parquet", + ) + endpoint_cap = _endpoint_budget_cap(config) + selected_row, selection_audit = select_policy_result_ex_ante( + selection_results, + endpoint_budget_cap=endpoint_cap, + budget=float(design["budget"]), + min_budget_utilization=float(design["selection_min_budget_utilization"]), + ) + candidate_lookup = {candidate.candidate_id: candidate for candidate in candidates} + selected = candidate_lookup[str(selected_row["candidate_id"])] + incumbent = _match_candidate(candidates, config["incumbent_policy"]) + cap_stability = endpoint_cap_stability( + selection_results, + selected_candidate_id=selected.candidate_id, + endpoint_budget_cap=endpoint_cap, + budget=float(design["budget"]), + min_budget_utilization=float(design["selection_min_budget_utilization"]), + ) + + audit_grid_path: Path | None = None + audit_evaluation_path: Path | None = None + audit_selected_id: str | None = None + audit_evaluation = pd.DataFrame() + if not audit_panel.empty: + audit_grid_path = data_dir / "calibration_policy_audit_grid.parquet" + audit_results = _run_calibration_grid( + audit_panel, + candidates, + config=config, + output_path=audit_grid_path, + ) + audit_selected_row, _audit_selection = select_policy_result_ex_ante( + audit_results, + endpoint_budget_cap=endpoint_cap, + budget=float(design["budget"]), + min_budget_utilization=float(design["selection_min_budget_utilization"]), + ) + audit_selected_id = str(audit_selected_row["candidate_id"]) + audited_decisions = audit_panel.assign(_outcome=audit_outcomes.to_numpy(dtype=float)) + audit_evaluation = _evaluate_calibration_audit( + audited_decisions, + selected, + incumbent, + config=config, + period=audit_period, + ) + audit_evaluation_path = data_dir / "calibration_policy_holdout_audit.csv" + audit_evaluation.to_csv(audit_evaluation_path, index=False) + + oot_panel = load_policy_panel(config) + evaluation, allocations = _evaluate_fixed_policies( + oot_panel, + selected, + incumbent, + config=config, + ) + evaluation_path = data_dir / "calibration_selected_policy_oot_evaluation.csv" + allocation_path = data_dir / "calibration_selected_policy_full_oot_allocations.parquet" + evaluation.to_csv(evaluation_path, index=False) + allocations.to_parquet(allocation_path, index=False) + payload: dict[str, Any] = { + "schema_version": str(config["schema_version"]), + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "run_tag": run_tag, + "source_commit": _git_commit(), + "config_path": str(config_path.relative_to(ROOT)), + "config_sha256": hashlib.sha256(config_path.read_bytes()).hexdigest(), + "design": config["design"], + "calibration_metadata": calibration_metadata, + "recipe": { + "partition": recipe.partition, + "partition_probability_source": recipe.partition_probability_source, + "reference_target_alpha": recipe.reference_target_alpha, + "reference_used_alpha": recipe.reference_used_alpha, + }, + "grid_size": int(len(candidates)), + "selector_input_columns": list(_selector_input_frame(selection_panel).columns), + "selector_columns": list(selection_results.columns), + "selector_forbidden_columns_present": sorted( + FORBIDDEN_SELECTOR_COLUMNS.intersection(selection_results.columns) + ), + "selection_audit": selection_audit, + "endpoint_cap_stability": cap_stability, + "calibration_audit": { + "period": audit_period or None, + "outcome_free_selected_candidate_id": audit_selected_id, + "same_policy_selected": bool(audit_selected_id == selected.candidate_id), + "grid_path": ( + None if audit_grid_path is None else str(audit_grid_path.relative_to(ROOT)) + ), + "evaluation_path": ( + None + if audit_evaluation_path is None + else str(audit_evaluation_path.relative_to(ROOT)) + ), + "policy_evaluations": audit_evaluation.to_dict(orient="records"), + "claim_boundary": ( + "Independent post-selection decision audit; not a selected-set coverage theorem." + ), + }, + "selected_policy": selected.to_record(), + "selected_calibration_metrics": selected_row.to_dict(), + "incumbent_policy": incumbent.to_record(), + "evaluation_path": str(evaluation_path.relative_to(ROOT)), + "allocation_path": str(allocation_path.relative_to(ROOT)), + "oot_alignment": dict(oot_panel.attrs), + "contrasts": _contrast_payload(evaluation), + "claim_boundary": str(config["claim_boundary"]), + } + summary_path = model_dir / "calibration_selected_policy_summary.json" + write_json(summary_path, payload) + logger.info("Selected calibration-only policy: {}", selected.candidate_id) + logger.info("Wrote calibration-only policy summary to {}", summary_path) + return payload + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + args = parser.parse_args() + config_path = args.config if args.config.is_absolute() else ROOT / args.config + payload = run(config_path.resolve()) + print(json.dumps(payload["contrasts"], indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py b/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py new file mode 100644 index 0000000..5b71902 --- /dev/null +++ b/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py @@ -0,0 +1,291 @@ +"""Recompute the IJDS alpha grid from the frozen conformal recipe.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import pickle +import sys +from dataclasses import asdict +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import yaml +from loguru import logger + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from scripts.generate_conformal_intervals import ( # noqa: E402 + _build_probability_lookups, + _build_tuning_split, + _load_conformal_inputs, +) +from src.models.conformal import conditional_coverage_by_group # noqa: E402 +from src.models.conformal_alpha_grid import ( # noqa: E402 + ExactAlphaIntervals, + FrozenConformalRecipe, + alpha_interval_columns, + compute_exact_alpha_intervals, +) +from src.utils.script_helpers import resolve_repo_artifact_path, write_json # noqa: E402 + +DEFAULT_CONFIG = ROOT / "configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml" + + +def _utc_now() -> str: + return datetime.now(tz=UTC).isoformat() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Experiment config must contain a mapping.") + return payload + + +def _experiment_paths(run_tag: str) -> tuple[Path, Path]: + data_dir = ROOT / "data/processed/experiments/champion_reopen" / run_tag / "conformal" + model_dir = ROOT / "models/experiments/champion_reopen" / run_tag / "conformal" + data_dir.mkdir(parents=True, exist_ok=True) + model_dir.mkdir(parents=True, exist_ok=True) + return data_dir, model_dir + + +def _load_results_payload(path: Path) -> dict[str, Any]: + with path.open("rb") as handle: + payload = pickle.load(handle) + if not isinstance(payload, dict): + raise TypeError("Frozen conformal results must contain a mapping.") + return payload + + +def _coverage_summary( + result: ExactAlphaIntervals, + *, + y_true: np.ndarray, + grades: pd.Series, +) -> dict[str, Any]: + covered = (y_true >= result.low) & (y_true <= result.high) + partition_metrics = conditional_coverage_by_group( + y_true, + np.column_stack([result.low, result.high]), + result.partition_labels, + ) + grade_metrics = conditional_coverage_by_group( + y_true, + np.column_stack([result.low, result.high]), + grades, + ) + return { + "target_alpha": result.target_alpha, + "used_alpha": result.used_alpha, + "target_coverage": 1.0 - result.target_alpha, + "empirical_coverage": float(covered.mean()), + "coverage_gap": float(covered.mean() - (1.0 - result.target_alpha)), + "avg_width": float(np.mean(result.high - result.low)), + "median_width": float(np.median(result.high - result.low)), + "min_partition_coverage": float(partition_metrics["coverage"].min()), + "min_grade_coverage": float(grade_metrics["coverage"].min()), + "high_endpoint_mean": float(result.high.mean()), + "high_endpoint_min": float(result.high.min()), + "high_endpoint_p01": float(np.quantile(result.high, 0.01)), + "high_endpoint_p10": float(np.quantile(result.high, 0.10)), + "high_endpoint_at_one_rate": float(np.mean(result.high >= 1.0 - 1e-12)), + "partition_count": int(result.partition_labels.nunique()), + "group_quantiles": { + str(key): float(value) + for key, value in result.diagnostics.get("group_quantiles", {}).items() + }, + } + + +def _base_grid_frame(source_intervals: pd.DataFrame) -> pd.DataFrame: + preferred = [ + "_row_number", + "id", + "y_true", + "grade", + "loan_amnt", + "temporal_segment", + ] + columns = [column for column in preferred if column in source_intervals.columns] + return source_intervals.loc[:, columns].copy() + + +def _add_alpha_result(frame: pd.DataFrame, result: ExactAlphaIntervals) -> None: + low_column, high_column = alpha_interval_columns(result.target_alpha) + frame[low_column] = result.low + frame[high_column] = result.high + + +def _replay_differences( + result: ExactAlphaIntervals, + source_intervals: pd.DataFrame, +) -> dict[str, float]: + return { + "point_max_abs": float( + np.max(np.abs(result.point - source_intervals["y_pred"].to_numpy(dtype=float))) + ), + "low_max_abs": float( + np.max(np.abs(result.low - source_intervals["pd_low_90"].to_numpy(dtype=float))) + ), + "high_max_abs": float( + np.max(np.abs(result.high - source_intervals["pd_high_90"].to_numpy(dtype=float))) + ), + } + + +def run(config_path: Path) -> dict[str, Any]: + config = _load_config(config_path) + source = config["source"] + design = config["design"] + run_tag = str(config["run_tag"]) + results_path = resolve_repo_artifact_path(source["conformal_results_path"], root=ROOT) + intervals_path = resolve_repo_artifact_path(source["conformal_intervals_path"], root=ROOT) + os.environ["UPSTREAM_CANONICAL_RUN_TAG"] = str(source["upstream_canonical_run_tag"]) + + results_payload = _load_results_payload(results_path) + recipe = FrozenConformalRecipe.from_results_payload(results_payload) + inputs = _load_conformal_inputs( + calibration_fraction=recipe.calibration_fraction, + calibrator_override_path=( + str(results_payload.get("calibrator_override_path", "")).strip() or None + ), + ) + split = _build_tuning_split( + cal_df=inputs.cal_df, + test_df=inputs.test_df, + X_cal=inputs.X_cal, + y_cal=inputs.y_cal, + group_cal_base=inputs.group_cal_base, + y_prob_cal_raw=inputs.y_prob_cal_raw, + tuning_holdout_ratio=recipe.tuning_holdout_ratio, + tuning_random_state=recipe.tuning_random_state, + ) + probability_fit, _probability_tune, probability_test = _build_probability_lookups( + inputs, + split, + ) + source_intervals = pd.read_parquet(intervals_path) + if len(source_intervals) != len(inputs.y_test): + raise ValueError("Source conformal intervals and evaluation rows differ in length.") + + grid = _base_grid_frame(source_intervals) + grid["y_pred"] = np.nan + alpha_summaries: list[dict[str, Any]] = [] + results: dict[float, ExactAlphaIntervals] = {} + for alpha in [float(value) for value in design["alpha_grid"]]: + result = compute_exact_alpha_intervals( + recipe=recipe, + target_alpha=alpha, + y_cal=split.y_cal_fit, + interval_probability_cal=probability_fit["calibrated"], + interval_probability_eval=probability_test["calibrated"], + partition_probability_cal=probability_fit[recipe.partition_probability_source], + partition_probability_eval=probability_test[recipe.partition_probability_source], + base_groups_cal=split.group_cal_fit_base, + base_groups_eval=inputs.group_test_base, + issue_dates_eval=split.issue_test, + ) + results[alpha] = result + if grid["y_pred"].isna().all(): + grid["y_pred"] = result.point + elif not np.array_equal(grid["y_pred"].to_numpy(dtype=float), result.point): + raise AssertionError("Point predictions changed across alpha levels.") + _add_alpha_result(grid, result) + summary = _coverage_summary( + result, + y_true=inputs.y_test.to_numpy(dtype=float), + grades=inputs.group_test_base.reset_index(drop=True), + ) + alpha_summaries.append(summary) + logger.info( + "Exact alpha={:.3f} (used={:.4f}): coverage={:.4f}, width={:.4f}, high=1 rate={:.2%}", + alpha, + result.used_alpha, + summary["empirical_coverage"], + summary["avg_width"], + summary["high_endpoint_at_one_rate"], + ) + + reference_alpha = recipe.reference_target_alpha + reference_result = next( + (result for alpha, result in results.items() if np.isclose(alpha, reference_alpha)), + None, + ) + if reference_result is None: + raise ValueError("Alpha grid must contain the recipe reference target alpha.") + replay = _replay_differences(reference_result, source_intervals) + tolerance = float(design["replay_tolerance"]) + replay["tolerance"] = tolerance + replay["pass"] = bool(max(replay.values()) <= tolerance) + if not replay["pass"]: + raise AssertionError(f"Frozen 90% interval replay drifted: {replay}") + + data_dir, model_dir = _experiment_paths(run_tag) + grid_path = data_dir / "exact_alpha_grid.parquet" + summary_path = model_dir / "exact_alpha_grid_summary.json" + grid.to_parquet(grid_path, index=False) + summary_payload: dict[str, Any] = { + "schema_version": str(config["schema_version"]), + "generated_at_utc": _utc_now(), + "run_tag": run_tag, + "source_commit": _git_commit(), + "config_path": str(config_path.relative_to(ROOT)), + "config_sha256": _sha256(config_path), + "source": { + **source, + "conformal_results_sha256": _sha256(results_path), + "conformal_intervals_sha256": _sha256(intervals_path), + }, + "recipe": asdict(recipe), + "alpha_mapping": str(design["alpha_mapping"]), + "reference_replay": replay, + "alpha_summaries": alpha_summaries, + "grid_path": str(grid_path.relative_to(ROOT)), + "grid_rows": int(len(grid)), + "claim_boundary": str(config["claim_boundary"]), + } + write_json(summary_path, summary_payload) + logger.info("Wrote exact alpha grid to {}", grid_path) + logger.info("Wrote exact alpha summary to {}", summary_path) + return summary_payload + + +def _git_commit() -> str: + import subprocess + + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + args = parser.parse_args() + config_path = args.config if args.config.is_absolute() else ROOT / args.config + run(config_path.resolve()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/experiments/run_tabpfn_tabprep_full.py b/scripts/experiments/run_tabpfn_tabprep_full.py index 4e9d7a2..d3787a5 100644 --- a/scripts/experiments/run_tabpfn_tabprep_full.py +++ b/scripts/experiments/run_tabpfn_tabprep_full.py @@ -8,6 +8,7 @@ import argparse import gc +import importlib import json import os import sys @@ -184,9 +185,8 @@ def main() -> None: del train, generated_train gc.collect() - from tabpfn import TabPFNClassifier - - clf = TabPFNClassifier( + tabpfn_classifier = _tabpfn_classifier_class() + clf = tabpfn_classifier( n_estimators=int(config["tabpfn"]["n_estimators"]), categorical_features_indices=categorical_indices, device=str(config["tabpfn"]["device"]), @@ -292,10 +292,10 @@ def _load_config(path: Path) -> dict[str, Any]: def _check_tabpfn_access(config: Mapping[str, Any]) -> None: """Fail before loading CRPTO data if TabPFN-3 weights are not accessible.""" from sklearn.datasets import make_classification - from tabpfn import TabPFNClassifier + tabpfn_classifier = _tabpfn_classifier_class() x, y = make_classification(n_samples=80, n_features=8, random_state=42) - clf = TabPFNClassifier( + clf = tabpfn_classifier( n_estimators=1, device=str(config["tabpfn"]["device"]), ignore_pretraining_limits=True, @@ -317,12 +317,27 @@ def _check_tabpfn_access(config: Mapping[str, Any]) -> None: ) from exc +def _tabpfn_classifier_class() -> Any: + """Return the optional TabPFN classifier class with a readable error.""" + try: + module = importlib.import_module("tabpfn") + except ImportError as exc: + raise RuntimeError( + "TabPFN + TabPrep is an isolated challenger and requires the optional " + "TabPFN package plus PriorLabs model access. Install/authorize TabPFN " + "before running this experiment." + ) from exc + return module.TabPFNClassifier + + def _preflight_dataset_limits(*, config: Mapping[str, Any], force: bool) -> None: """Check full-data shape against public TabPFN-3 limits.""" parquet_file = pq.ParquetFile(config["data"]["train_path"]) train_rows = int(parquet_file.metadata.num_rows) schema_columns = list(parquet_file.schema_arrow.names) - schema_frame = pd.DataFrame(columns=schema_columns) + schema_frame = pd.DataFrame( + {str(column): pd.Series(dtype="object") for column in schema_columns} + ) feature_config = load_feature_config( yaml_path=Path(config["data"]["feature_config_path"]), prefer="yaml", diff --git a/scripts/experiments/run_tabprep_feature_selection_catboost.py b/scripts/experiments/run_tabprep_feature_selection_catboost.py index b024b73..8a1212a 100644 --- a/scripts/experiments/run_tabprep_feature_selection_catboost.py +++ b/scripts/experiments/run_tabprep_feature_selection_catboost.py @@ -190,9 +190,10 @@ def main() -> None: pool_features, feature_config=feature_config, ) - core_features = _resolve_core_features(feature_config, train.columns) - catboost_features = _resolve_catboost_features(feature_config, train.columns) - woe_features = _resolve_woe_features(feature_config, train.columns) + train_columns = [str(column) for column in train.columns] + core_features = _resolve_core_features(feature_config, train_columns) + catboost_features = _resolve_catboost_features(feature_config, train_columns) + woe_features = _resolve_woe_features(feature_config, train_columns) train_fit, train_val = temporal_train_val_split( train, diff --git a/scripts/export_crpto_tables.py b/scripts/export_crpto_tables.py index 4f41837..bdb0ac5 100644 --- a/scripts/export_crpto_tables.py +++ b/scripts/export_crpto_tables.py @@ -18,8 +18,8 @@ from typing import Any import pandas as pd -from analyze_crpto_evidence import build_p1_evidence # type: ignore[import-not-found] +from scripts.analyze_crpto_evidence import build_p1_evidence from src.utils.script_helpers import first_existing, load_json, policy_matches, write_table ROOT = Path(__file__).resolve().parents[1] @@ -76,7 +76,7 @@ def _table0_key_metrics( ("alpha01_gamma_cp", champ["alpha01_gamma_cp"]), ("alpha01_violation", champ["alpha01_violation"]), ] - return pd.DataFrame(rows, columns=["metric", "value"]) + return pd.DataFrame([{"metric": metric, "value": value} for metric, value in rows]) def _table1_robustness_summary( diff --git a/scripts/generate_conformal_intervals.py b/scripts/generate_conformal_intervals.py index 9a01e9c..165d18a 100644 --- a/scripts/generate_conformal_intervals.py +++ b/scripts/generate_conformal_intervals.py @@ -11,8 +11,10 @@ import pickle import shutil import time +from collections.abc import Iterable from dataclasses import dataclass from datetime import UTC, datetime +from itertools import product from pathlib import Path from typing import Any @@ -57,6 +59,56 @@ TARGET_COL = "default_flag" GROUP_COL = "grade" +PartitionCache = dict[ + tuple[str, str, str, int, str, int], + tuple[pd.Series, pd.Series, dict[str, Any]], +] + + +def _empty_coverage_floor_report() -> pd.DataFrame: + return pd.DataFrame( + { + "group": pd.Series(dtype="object"), + "coverage_before": pd.Series(dtype="float64"), + "coverage_after": pd.Series(dtype="float64"), + "target_coverage": pd.Series(dtype="float64"), + "multiplier": pd.Series(dtype="float64"), + "adjusted": pd.Series(dtype="bool"), + } + ) + + +def _empty_temporal_segment_report() -> pd.DataFrame: + return pd.DataFrame( + { + "segment": pd.Series(dtype="object"), + "support_n": pd.Series(dtype="int64"), + "coverage_before": pd.Series(dtype="float64"), + "coverage_after": pd.Series(dtype="float64"), + "target_coverage": pd.Series(dtype="float64"), + "min_segment_size": pd.Series(dtype="int64"), + "multiplier": pd.Series(dtype="float64"), + "adjusted": pd.Series(dtype="bool"), + } + ) + + +def _empty_shrinkback_report() -> pd.DataFrame: + return pd.DataFrame( + { + "stage": pd.Series(dtype="object"), + "factor_scope": pd.Series(dtype="object"), + "factor_key": pd.Series(dtype="object"), + "candidate_factor": pd.Series(dtype="float64"), + "accepted": pd.Series(dtype="bool"), + "coverage": pd.Series(dtype="float64"), + "min_group_coverage": pd.Series(dtype="float64"), + "avg_width": pd.Series(dtype="float64"), + "winkler_90": pd.Series(dtype="float64"), + "max_monthly_gap": pd.Series(dtype="float64"), + "stability_over_time": pd.Series(dtype="float64"), + } + ) def _utc_now() -> str: @@ -150,6 +202,73 @@ class ConformalTuningSelection: best_cfg: dict[str, Any] +@dataclass(frozen=True) +class ConformalTuningSearch: + """Raw tuning rows plus cached partition labels reusable for final intervals.""" + + tuning_rows: list[dict[str, Any]] + partition_cache: PartitionCache + + +@dataclass(frozen=True) +class GlobalRebalanceResult: + """Final 90% intervals and diagnostics after optional global rebalance.""" + + y_intervals: np.ndarray + metrics: dict[str, Any] + group_metrics: pd.DataFrame + factor: float + diagnostics: dict[str, float | bool] + + +@dataclass(frozen=True) +class ConformalArtifactTables: + """Final tabular conformal artifacts ready for persistence.""" + + intervals: pd.DataFrame + group_metrics: pd.DataFrame + width_attribution: pd.DataFrame + + +@dataclass(frozen=True) +class ConformalEvidence90: + """Adjusted 90% interval evidence and learned coverage-floor policy.""" + + y_eval: pd.Series + eval_groups: pd.Series + eval_issue: pd.Series + y_pred: np.ndarray + y_intervals: np.ndarray + diag: dict[str, Any] + metrics: dict[str, Any] + group_metrics: pd.DataFrame + y_pred_tune: np.ndarray + y_intervals_tune_base: np.ndarray + tune_metrics_before_floor: dict[str, Any] + tune_metrics_after_floor: dict[str, Any] + tune_metrics_after_temporal_floor: dict[str, Any] + group_multipliers: dict[str, float] + coverage_floor_report: pd.DataFrame + temporal_segment_multipliers: dict[str, float] + temporal_segment_report: pd.DataFrame + shrinkback_report: pd.DataFrame + eval_temporal_segments: pd.Series | None + width_attr_rows: list[dict[str, Any]] + global_rebalance_factor: float + global_rebalance_diagnostics: dict[str, float | bool] + + +@dataclass(frozen=True) +class ConformalEvidence95: + """95% interval evidence after applying the learned 90% adjustment policy.""" + + y_pred: np.ndarray + y_intervals: np.ndarray + diag: dict[str, Any] + metrics: dict[str, Any] + group_metrics: pd.DataFrame + + def _resolve_artifact_paths(namespace: str | None = None) -> dict[str, Path]: if namespace: ns = str(namespace).strip().replace("/", "_") @@ -306,62 +425,83 @@ def _load_calibrator(calibrator_override_path: str | None = None) -> Any | None: return calibrator -def _resolve_features( - model: CatBoostClassifier, - cal_df: pd.DataFrame, - test_df: pd.DataFrame, -) -> tuple[list[str], list[str]]: - """Resolve feature list, preferring explicit contract then model metadata.""" - contract = _load_active_contract() - if isinstance(contract, dict): - contract_features = contract.get("feature_names", []) - contract_categorical = contract.get("categorical_features", []) - if contract_features: - categorical = [c for c in contract_categorical if c in contract_features] - logger.info( - f"Using {len(contract_features)} contract features ({len(categorical)} categorical) " - f"from {contract.get('_contract_path', CONTRACT_PATH)}" - ) - return list(contract_features), categorical +def _features_from_contract(contract: dict[str, Any] | None) -> tuple[list[str], list[str]] | None: + if not isinstance(contract, dict): + return None + contract_features = list(contract.get("feature_names", []) or []) + if not contract_features: + return None + contract_categorical = list(contract.get("categorical_features", []) or []) + categorical = [feature for feature in contract_categorical if feature in contract_features] + logger.info( + f"Using {len(contract_features)} contract features ({len(categorical)} categorical) " + f"from {contract.get('_contract_path', CONTRACT_PATH)}" + ) + return contract_features, categorical + +def _features_from_model(model: CatBoostClassifier) -> tuple[list[str], list[str]] | None: model_features = list(getattr(model, "feature_names_", []) or []) - if model_features: - cat_idxs = set(model.get_cat_feature_indices()) - categorical = [f for i, f in enumerate(model_features) if i in cat_idxs] - logger.info( - f"Using {len(model_features)} model-native features ({len(categorical)} categorical)" - ) - return model_features, categorical + if not model_features: + return None + categorical_indexes = set(model.get_cat_feature_indices()) + categorical = [ + feature for index, feature in enumerate(model_features) if index in categorical_indexes + ] + logger.info( + f"Using {len(model_features)} model-native features ({len(categorical)} categorical)" + ) + return model_features, categorical + - # Fallback path if model metadata is unavailable. - feature_cfg_path = Path("data/processed/feature_config.yml") - feature_cfg: dict[str, Any] = {} +def _load_fallback_feature_config(feature_cfg_path: Path) -> dict[str, Any]: try: - feature_cfg = load_feature_config_artifact( + return load_feature_config_artifact( pickle_path=feature_cfg_path.with_suffix(".pkl"), yaml_path=feature_cfg_path, prefer="yaml", ) except (FileNotFoundError, TypeError) as exc: logger.warning(f"Unable to load fallback feature_config from {feature_cfg_path}: {exc}") + return {} + +def _features_from_fallback_config( + *, + cal_df: pd.DataFrame, + test_df: pd.DataFrame, +) -> tuple[list[str], list[str]]: + feature_cfg = _load_fallback_feature_config(Path("data/processed/feature_config.yml")) catboost_features = feature_cfg.get("CATBOOST_FEATURES", []) categorical = feature_cfg.get("CATEGORICAL_FEATURES", []) - - features = [c for c in catboost_features if c in cal_df.columns and c in test_df.columns] + features = [ + name for name in catboost_features if name in cal_df.columns and name in test_df.columns + ] if not features: from src.models.pd_model import get_available_features - features = [c for c in get_available_features(cal_df) if c in test_df.columns] - + features = [name for name in get_available_features(cal_df) if name in test_df.columns] if not features: raise ValueError("Unable to resolve feature list for conformal generation.") - - categorical = [c for c in categorical if c in features] + categorical = [name for name in categorical if name in features] logger.info(f"Using {len(features)} features ({len(categorical)} categorical)") return features, categorical +def _resolve_features( + model: CatBoostClassifier, + cal_df: pd.DataFrame, + test_df: pd.DataFrame, +) -> tuple[list[str], list[str]]: + """Resolve feature list, preferring explicit contract then model metadata.""" + contract = _load_active_contract() + return ( + _features_from_contract(contract) + or _features_from_model(model) + or _features_from_fallback_config(cal_df=cal_df, test_df=test_df) + ) + + def _build_feature_matrix( df: pd.DataFrame, features: list[str], @@ -383,6 +523,33 @@ def _build_feature_matrix( return X +def _align_contract_matrix( + matrix: pd.DataFrame, + reference_frame: pd.DataFrame, + *, + path: Path, + split_name: str, +) -> pd.DataFrame: + if len(matrix) == len(reference_frame): + return matrix + if "id" in matrix.columns and "id" in reference_frame.columns: + indexed = matrix.set_index("id", drop=False) + wanted = reference_frame["id"].tolist() + missing_ids = [value for value in wanted if value not in indexed.index] + if missing_ids: + raise KeyError( + f"Contract model matrix {path} missing {len(missing_ids)} ids; " + f"first missing={missing_ids[:5]}" + ) + return indexed.loc[wanted].reset_index(drop=True) + if len(matrix) > len(reference_frame): + return matrix.tail(len(reference_frame)).reset_index(drop=True) + raise ValueError( + f"Contract model matrix row count mismatch for {split_name}: " + f"matrix={len(matrix)}, reference={len(reference_frame)}" + ) + + def _load_contract_matrix( *, contract: dict[str, Any] | None, @@ -403,24 +570,12 @@ def _load_contract_matrix( if not path.exists(): raise FileNotFoundError(f"Contract model matrix not found for {split_name}: {path}") matrix = pd.read_parquet(path) - if len(matrix) != len(reference_frame): - if "id" in matrix.columns and "id" in reference_frame.columns: - indexed = matrix.set_index("id", drop=False) - wanted = reference_frame["id"].tolist() - missing_ids = [value for value in wanted if value not in indexed.index] - if missing_ids: - raise KeyError( - f"Contract model matrix {path} missing {len(missing_ids)} ids; " - f"first missing={missing_ids[:5]}" - ) - matrix = indexed.loc[wanted].reset_index(drop=True) - elif len(matrix) > len(reference_frame): - matrix = matrix.tail(len(reference_frame)).reset_index(drop=True) - else: - raise ValueError( - f"Contract model matrix row count mismatch for {split_name}: " - f"matrix={len(matrix)}, reference={len(reference_frame)}" - ) + matrix = _align_contract_matrix( + matrix, + reference_frame, + path=path, + split_name=split_name, + ) missing = [feature for feature in features if feature not in matrix.columns] if missing: raise KeyError( @@ -566,6 +721,29 @@ def _load_conformal_inputs( ) +def _dedupe_string_candidates( + values: Iterable[Any], + *, + default: tuple[str, ...], + lower: bool = False, +) -> tuple[str, ...]: + normalized = [] + for value in values: + token = str(value).strip() + if token: + normalized.append(token.lower() if lower else token) + return tuple(dict.fromkeys(normalized)) or default + + +def _positive_int_candidates( + values: Iterable[Any], + *, + default: tuple[int, ...], +) -> tuple[int, ...]: + resolved = tuple(int(value) for value in values if int(value) > 0) + return resolved or default + + def _resolve_tuning_grid( *, partition: str, @@ -577,35 +755,26 @@ def _resolve_tuning_grid( scaled_scores_options: tuple[bool, ...], ) -> ConformalTuningGrid: """Normalize and de-duplicate Mondrian tuning candidates.""" - resolved_partitions = tuple( - dict.fromkeys( - [ - str(token).strip() - for token in (partition_candidates or (partition,)) - if str(token).strip() - ] - ) - ) or (str(partition).strip() or "grade",) - resolved_probability_sources = tuple( - dict.fromkeys( - str(source).strip().lower() - for source in partition_probability_sources - if str(source).strip() - ) - ) or ("raw",) - resolved_score_bins = tuple(int(x) for x in n_score_bins_candidates if int(x) > 0) or (10,) - resolved_fallback_modes = tuple( - dict.fromkeys( - str(mode_name).strip().lower() for mode_name in fallback_modes if str(mode_name).strip() - ) - ) or ("grade_then_global",) - resolved_score_families = tuple( - dict.fromkeys( - str(scale_name).strip().lower() - for scale_name in score_scale_families - if str(scale_name).strip() - ) - ) or ("none",) + resolved_partitions = _dedupe_string_candidates( + partition_candidates or (partition,), + default=(str(partition).strip() or "grade",), + ) + resolved_probability_sources = _dedupe_string_candidates( + partition_probability_sources, + default=("raw",), + lower=True, + ) + resolved_score_bins = _positive_int_candidates(n_score_bins_candidates, default=(10,)) + resolved_fallback_modes = _dedupe_string_candidates( + fallback_modes, + default=("grade_then_global",), + lower=True, + ) + resolved_score_families = _dedupe_string_candidates( + score_scale_families, + default=("none",), + lower=True, + ) return ConformalTuningGrid( partition_candidates=resolved_partitions, partition_probability_sources=resolved_probability_sources, @@ -725,472 +894,616 @@ def _select_best_tuning_config( ) -def _parse_float_tuple(raw: str) -> tuple[float, ...]: - values = [float(token.strip()) for token in str(raw).split(",") if token.strip()] - if not values: - raise ValueError("Expected at least one float value.") - return tuple(values) +def _log_tuning_split(inputs: ConformalInputs, tuning_split: ConformalTuningSplit) -> None: + logger.info( + "Calibration split for conformal tuning: " + f"fit={len(tuning_split.X_cal_fit):,}, holdout={len(tuning_split.X_tune):,}, " + f"holdout_ratio={len(tuning_split.X_tune) / max(len(inputs.X_cal), 1):.2%}" + ) + if "issue_d" not in inputs.cal_df.columns: + return + fit_issue = tuning_split.issue_cal.iloc[tuning_split.idx_cal_fit] + tune_issue = tuning_split.issue_tune + if fit_issue.notna().any() and tune_issue.notna().any(): + logger.info( + "Calibration split date ranges: " + f"fit_max={fit_issue.max():%Y-%m}, " + f"holdout_min={tune_issue.min():%Y-%m}, " + f"holdout_max={tune_issue.max():%Y-%m}" + ) -def _parse_int_tuple(raw: str) -> tuple[int, ...]: - values = [int(token.strip()) for token in str(raw).split(",") if token.strip()] - if not values: - raise ValueError("Expected at least one integer value.") - return tuple(values) +def _build_probability_lookups( + inputs: ConformalInputs, + tuning_split: ConformalTuningSplit, +) -> tuple[dict[str, np.ndarray], dict[str, np.ndarray], dict[str, np.ndarray]]: + return ( + { + "raw": tuning_split.y_prob_cal_fit, + "calibrated": inputs.y_prob_calibrated[tuning_split.idx_cal_fit], + }, + { + "raw": tuning_split.y_prob_cal_tune, + "calibrated": inputs.y_prob_calibrated[tuning_split.idx_cal_tune], + }, + {"raw": inputs.y_prob_test_raw, "calibrated": inputs.y_prob_test_calibrated}, + ) -def _parse_bool_tuple(raw: str) -> tuple[bool, ...]: - values = [ - token.strip().lower() in {"1", "true", "yes", "y"} - for token in str(raw).split(",") - if token.strip() - ] - if not values: - raise ValueError("Expected at least one boolean value.") - return tuple(values) +def _tuning_total_candidates( + tuning_grid: ConformalTuningGrid, + *, + alpha_candidates_90: tuple[float, ...], + min_group_sizes: tuple[int, ...], +) -> int: + return ( + len(tuning_grid.partition_candidates) + * len(tuning_grid.partition_probability_sources) + * len(tuning_grid.n_score_bins_candidates) + * len(tuning_grid.fallback_modes) + * len(alpha_candidates_90) + * len(tuning_grid.scaled_scores_options) + * len(tuning_grid.score_scale_families) + * len(min_group_sizes) + ) -def _parse_str_tuple(raw: str) -> tuple[str, ...]: - values = tuple(token.strip() for token in str(raw).split(",") if token.strip()) - if not values: - raise ValueError("Expected at least one string value.") - return values +def _cached_partition_labels( + *, + cache: PartitionCache, + eval_scope: str, + partition_candidate: str, + partition_probability_source: str, + n_score_bins: int, + fallback_mode: str, + min_group_size: int, + prob_fit_lookup: dict[str, np.ndarray], + prob_tune_lookup: dict[str, np.ndarray], + prob_test_lookup: dict[str, np.ndarray], + group_cal_fit_base: pd.Series, + group_tune_base: pd.Series, + group_test_base: pd.Series, +) -> tuple[pd.Series, pd.Series, dict[str, Any]]: + key = ( + str(eval_scope), + str(partition_candidate), + str(partition_probability_source), + int(n_score_bins), + str(fallback_mode), + int(min_group_size), + ) + if key in cache: + return cache[key] + if eval_scope == "test": + y_prob_eval = prob_test_lookup[partition_probability_source] + base_groups_eval = group_test_base + elif eval_scope == "tune": + y_prob_eval = prob_tune_lookup[partition_probability_source] + base_groups_eval = group_tune_base + else: + raise ValueError(f"Unsupported cached partition eval_scope: {eval_scope}") + payload = build_mondrian_partition_labels( + y_prob_cal=prob_fit_lookup[partition_probability_source], + y_prob_eval=y_prob_eval, + partition=partition_candidate, + base_groups_cal=group_cal_fit_base, + base_groups_eval=base_groups_eval, + n_score_bins=n_score_bins, + min_group_size=min_group_size, + fallback_mode=fallback_mode, + ) + cache[key] = payload + return payload -def main( - alpha_target_90: float = 0.10, - alpha_95: float = 0.05, - alpha_candidates_90: tuple[float, ...] = (0.10, 0.095, 0.09, 0.085, 0.08), - alpha_candidates_95: tuple[float, ...] = (0.05,), - min_group_sizes: tuple[int, ...] = (200, 500, 1000, 2000), - min_group_coverage_target: float = 0.88, - group_coverage_floor_target_90: float = 0.92, - max_width_budget_90: float | None = 0.80, - coverage_guardband_90: float = 0.015, - min_group_guardband_90: float = 0.0, - tuning_holdout_ratio: float = 0.20, - tuning_random_state: int = 42, - temporal_segment_floor_enabled: bool = True, - temporal_segment_freq: str = "Q", - temporal_segment_min_size: int = 250, - global_rebalance_enabled: bool = False, - global_rebalance_min_factor: float = 0.75, - global_rebalance_max_factor: float = 1.05, - global_rebalance_step: float = 0.01, - partition: str = "grade", - partition_candidates: tuple[str, ...] | None = None, - group_coverage_floor_enabled: bool = True, - shrinkback_enabled: bool = False, - group_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), - temporal_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), - artifact_namespace: str | None = None, - scaled_scores_options: tuple[bool, ...] = (True, False), - score_scale_families: tuple[str, ...] = ("bernoulli_sqrt", "none"), - partition_probability_sources: tuple[str, ...] = ("raw",), - n_score_bins_candidates: tuple[int, ...] = (10,), - fallback_modes: tuple[str, ...] = ("grade_then_global",), - calibration_fraction: float = 1.0, - evaluation_scope: str = "test", - mode: str = "search", - replay_manifest_path: str | None = None, - calibrator_override_path: str | None = None, - run_tag: str | None = None, -): - logger.info("Starting Mondrian conformal interval generation with 90% auto-tuning") - run_mode = str(mode or "search").strip().lower() or "search" - replay_cfg = manifest_section(load_replay_manifest(replay_manifest_path), "conformal") - if run_mode == "replay": - source_namespace = str(replay_cfg.get("source_namespace", "")).strip() - replay_mode = str(replay_cfg.get("replay_mode", "")).strip().lower() - if replay_mode != "restore_blessed_namespace" or not source_namespace: - raise ValueError( - "Conformal replay requires source_namespace + restore_blessed_namespace." - ) - _restore_replay_namespace(source_namespace, run_tag=run_tag) - return +def _build_tuning_row( + *, + partition_candidate: str, + partition_probability_source: str, + n_score_bins: int, + fallback_mode: str, + alpha_target_90: float, + alpha_used: float, + scaled_scores: bool, + score_scale_family: str, + min_group_size: int, + target_coverage_90: float, + y_tune: pd.Series, + y_int: np.ndarray, + group_tune: pd.Series, + issue_tune: pd.Series, + partition_meta_candidate: dict[str, Any], +) -> dict[str, Any]: + y_tune_array = y_tune.to_numpy(dtype=float) + metrics = validate_coverage(y_tune_array, y_int, alpha_target_90, log_summary=False) + group_metrics = conditional_coverage_by_group(y_tune_array, y_int, group_tune) + temporal_metrics = temporal_stability_summary( + y_tune_array, + y_int, + issue_tune, + target_coverage=target_coverage_90, + freq="M", + ) + return { + "partition": str(partition_meta_candidate.get("partition", partition_candidate)), + "partition_probability_source": partition_probability_source, + "n_score_bins": int(n_score_bins), + "fallback_mode": str(partition_meta_candidate.get("fallback_mode", fallback_mode)), + "fallback_groups_n": len(partition_meta_candidate.get("fallback_groups", [])), + "alpha_target_90": alpha_target_90, + "alpha_used_90": alpha_used, + "scaled_scores": bool(scaled_scores), + "score_scale_family": str(score_scale_family), + "min_group_size": int(min_group_size), + "empirical_coverage": float(metrics["empirical_coverage"]), + "target_coverage": float(metrics["target_coverage"]), + "coverage_gap": float(metrics["coverage_gap"]), + "avg_interval_width": float(metrics["avg_interval_width"]), + "median_interval_width": float(metrics["median_interval_width"]), + "min_group_coverage": float(group_metrics["coverage"].min()), + "max_group_coverage": float(group_metrics["coverage"].max()), + "std_group_coverage": float(group_metrics["coverage"].std(ddof=0)), + "winkler_90": float(mean_winkler_score(y_tune_array, y_int, alpha=alpha_target_90)), + "max_monthly_gap": float(temporal_metrics["max_monthly_gap"]), + "stability_over_time": float(temporal_metrics["stability_over_time"]), + } - inputs = _load_conformal_inputs( - calibration_fraction=calibration_fraction, - calibrator_override_path=calibrator_override_path, + +def _run_tuning_search( + *, + tuning_grid: ConformalTuningGrid, + alpha_candidates_90: tuple[float, ...], + min_group_sizes: tuple[int, ...], + alpha_target_90: float, + target_coverage_90: float, + artifact_namespace: str | None, + prob_fit_lookup: dict[str, np.ndarray], + prob_tune_lookup: dict[str, np.ndarray], + prob_test_lookup: dict[str, np.ndarray], + interval_fit_pred: np.ndarray, + interval_tune_pred: np.ndarray, + y_cal_fit: pd.Series, + y_tune: pd.Series, + group_cal_fit_base: pd.Series, + group_tune_base: pd.Series, + group_test_base: pd.Series, + issue_tune: pd.Series, +) -> ConformalTuningSearch: + unsupported_sources = set(tuning_grid.partition_probability_sources) - set(prob_fit_lookup) + if unsupported_sources: + raise ValueError( + f"Unsupported partition_probability_source: {', '.join(sorted(unsupported_sources))}" + ) + + total = _tuning_total_candidates( + tuning_grid, + alpha_candidates_90=alpha_candidates_90, + min_group_sizes=min_group_sizes, ) - model_path = inputs.model_path - cal_df = inputs.cal_df - test_df = inputs.test_df - X_cal = inputs.X_cal - y_cal = inputs.y_cal - y_test = inputs.y_test - group_cal_base = inputs.group_cal_base - group_test_base = inputs.group_test_base - y_prob_cal_raw = inputs.y_prob_cal_raw - y_prob_test_raw = inputs.y_prob_test_raw - y_prob_calibrated = inputs.y_prob_calibrated - y_prob_test_calibrated = inputs.y_prob_test_calibrated - tuning_split = _build_tuning_split( - cal_df=cal_df, - test_df=test_df, - X_cal=X_cal, - y_cal=y_cal, - group_cal_base=group_cal_base, - y_prob_cal_raw=y_prob_cal_raw, - tuning_holdout_ratio=tuning_holdout_ratio, - tuning_random_state=tuning_random_state, + started = time.perf_counter() + status_path = ( + _resolve_artifact_paths(artifact_namespace)["models_dir"] + / "conformal_tuning_runtime_status.json" ) - idx_cal_fit = tuning_split.idx_cal_fit - idx_cal_tune = tuning_split.idx_cal_tune - X_cal_fit = tuning_split.X_cal_fit - y_cal_fit = tuning_split.y_cal_fit - X_tune = tuning_split.X_tune - y_tune = tuning_split.y_tune - y_prob_cal_fit = tuning_split.y_prob_cal_fit - y_prob_cal_tune = tuning_split.y_prob_cal_tune - group_cal_fit_base = tuning_split.group_cal_fit_base - group_tune_base = tuning_split.group_tune_base - issue_cal = tuning_split.issue_cal - issue_tune = tuning_split.issue_tune - issue_test = tuning_split.issue_test - logger.info( - "Calibration split for conformal tuning: " - f"fit={len(X_cal_fit):,}, holdout={len(X_tune):,}, " - f"holdout_ratio={len(X_tune) / max(len(X_cal), 1):.2%}" + _write_tuning_runtime_status( + status_path, + artifact_namespace=artifact_namespace, + phase="tuning_running", + completed=0, + total=total, + started_monotonic=started, ) - if "issue_d" in cal_df.columns: - fit_issue = issue_cal.iloc[idx_cal_fit] - tune_issue = issue_tune - if fit_issue.notna().any() and tune_issue.notna().any(): - logger.info( - "Calibration split date ranges: " - f"fit_max={fit_issue.max():%Y-%m}, " - f"holdout_min={tune_issue.min():%Y-%m}, " - f"holdout_max={tune_issue.max():%Y-%m}" - ) - target_coverage_90 = 1.0 - alpha_target_90 - group_coverage_floor_target_90 = max( - float(min_group_coverage_target), - float(group_coverage_floor_target_90), + tuning_rows: list[dict[str, Any]] = [] + partition_cache: PartitionCache = {} + candidates = product( + tuning_grid.partition_candidates, + tuning_grid.partition_probability_sources, + tuning_grid.n_score_bins_candidates, + tuning_grid.fallback_modes, + alpha_candidates_90, + tuning_grid.scaled_scores_options, + tuning_grid.score_scale_families, + min_group_sizes, ) - evaluation_scope_key = str(evaluation_scope or "test").strip().lower() or "test" - if evaluation_scope_key not in {"test", "holdout"}: - raise ValueError(f"Unsupported evaluation_scope: {evaluation_scope}") - tuning_grid = _resolve_tuning_grid( - partition=partition, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - scaled_scores_options=scaled_scores_options, - ) - partition_candidates = tuning_grid.partition_candidates - partition_probability_sources = tuning_grid.partition_probability_sources - n_score_bins_candidates = tuning_grid.n_score_bins_candidates - fallback_modes = tuning_grid.fallback_modes - score_scale_families = tuning_grid.score_scale_families - scaled_scores_options = tuning_grid.scaled_scores_options - tuning_rows: list[dict[str, Any]] = [] - tuning_total_candidates = ( - len(partition_candidates) - * len(partition_probability_sources) - * len(n_score_bins_candidates) - * len(fallback_modes) - * len(alpha_candidates_90) - * len(scaled_scores_options) - * len(score_scale_families) - * len(min_group_sizes) - ) - tuning_completed_candidates = 0 - tuning_started = time.perf_counter() - tuning_status_path = ( - _resolve_artifact_paths(artifact_namespace)["models_dir"] - / "conformal_tuning_runtime_status.json" - ) - _write_tuning_runtime_status( - tuning_status_path, - artifact_namespace=artifact_namespace, - phase="tuning_running", - completed=0, - total=tuning_total_candidates, - started_monotonic=tuning_started, - ) - - # Tune 90% interval config across candidate Mondrian partitions. - prob_fit_lookup = {"raw": y_prob_cal_fit, "calibrated": y_prob_calibrated[idx_cal_fit]} - prob_tune_lookup = {"raw": y_prob_cal_tune, "calibrated": y_prob_calibrated[idx_cal_tune]} - prob_test_lookup = {"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated} - interval_fit_pred = y_prob_calibrated[idx_cal_fit] - interval_tune_pred = y_prob_calibrated[idx_cal_tune] - interval_test_pred = y_prob_test_calibrated - partition_cache: dict[ - tuple[str, str, int, str, int, str], - tuple[pd.Series, pd.Series, dict[str, Any]], - ] = {} - - def _cached_partition_labels( - *, - eval_scope: str, - partition_candidate: str, - partition_probability_source: str, - n_score_bins: int, - fallback_mode: str, - min_group_size: int, - ) -> tuple[pd.Series, pd.Series, dict[str, Any]]: - key = ( - str(eval_scope), - str(partition_candidate), - str(partition_probability_source), - int(n_score_bins), - str(fallback_mode), - int(min_group_size), - ) - if key in partition_cache: - return partition_cache[key] - if eval_scope == "test": - y_prob_eval = prob_test_lookup[partition_probability_source] - base_groups_eval = group_test_base - elif eval_scope == "tune": - y_prob_eval = prob_tune_lookup[partition_probability_source] - base_groups_eval = group_tune_base - else: - raise ValueError(f"Unsupported cached partition eval_scope: {eval_scope}") - payload = build_mondrian_partition_labels( - y_prob_cal=prob_fit_lookup[partition_probability_source], - y_prob_eval=y_prob_eval, - partition=partition_candidate, - base_groups_cal=group_cal_fit_base, - base_groups_eval=base_groups_eval, + for completed, candidate in enumerate(candidates, start=1): + ( + partition_candidate, + partition_probability_source, + n_score_bins, + fallback_mode, + alpha_used, + scaled_scores, + score_scale_family, + min_group_size, + ) = candidate + group_cal_fit, group_tune, partition_meta_candidate = _cached_partition_labels( + cache=partition_cache, + eval_scope="tune", + partition_candidate=partition_candidate, + partition_probability_source=partition_probability_source, n_score_bins=n_score_bins, - min_group_size=min_group_size, fallback_mode=fallback_mode, + min_group_size=min_group_size, + prob_fit_lookup=prob_fit_lookup, + prob_tune_lookup=prob_tune_lookup, + prob_test_lookup=prob_test_lookup, + group_cal_fit_base=group_cal_fit_base, + group_tune_base=group_tune_base, + group_test_base=group_test_base, + ) + _y_pred, y_int, _diag = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=interval_fit_pred, + y_test_pred=interval_tune_pred, + y_cal=y_cal_fit, + group_cal=group_cal_fit, + group_test=group_tune, + alpha=alpha_used, + min_group_size=min_group_size, + scaled_scores=scaled_scores, + score_scale_family=score_scale_family, + log_summary=False, + ) + tuning_rows.append( + _build_tuning_row( + partition_candidate=partition_candidate, + partition_probability_source=partition_probability_source, + n_score_bins=n_score_bins, + fallback_mode=fallback_mode, + alpha_target_90=alpha_target_90, + alpha_used=alpha_used, + scaled_scores=scaled_scores, + score_scale_family=score_scale_family, + min_group_size=min_group_size, + target_coverage_90=target_coverage_90, + y_tune=y_tune, + y_int=y_int, + group_tune=group_tune, + issue_tune=issue_tune, + partition_meta_candidate=partition_meta_candidate, + ) ) - partition_cache[key] = payload - return payload - - for partition_candidate in partition_candidates: - for partition_probability_source in partition_probability_sources: - if partition_probability_source not in prob_fit_lookup: - raise ValueError( - f"Unsupported partition_probability_source: {partition_probability_source}" - ) - for n_score_bins in n_score_bins_candidates: - for fallback_mode in fallback_modes: - for alpha_used in alpha_candidates_90: - for scaled_scores in scaled_scores_options: - for score_scale_family in score_scale_families: - for min_group_size in min_group_sizes: - group_cal_fit, group_tune, partition_meta_candidate = ( - _cached_partition_labels( - eval_scope="tune", - partition_candidate=partition_candidate, - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins, - fallback_mode=fallback_mode, - min_group_size=min_group_size, - ) - ) - y_pred, y_int, _diag = ( - create_pd_intervals_mondrian_from_predictions( - y_cal_pred=interval_fit_pred, - y_test_pred=interval_tune_pred, - y_cal=y_cal_fit, - group_cal=group_cal_fit, - group_test=group_tune, - alpha=alpha_used, - min_group_size=min_group_size, - scaled_scores=scaled_scores, - score_scale_family=score_scale_family, - log_summary=False, - ) - ) - - metrics = validate_coverage( - y_tune.to_numpy(dtype=float), - y_int, - alpha_target_90, - log_summary=False, - ) - g_metrics = conditional_coverage_by_group( - y_tune.to_numpy(dtype=float), y_int, group_tune - ) - temporal_metrics = temporal_stability_summary( - y_tune.to_numpy(dtype=float), - y_int, - issue_tune, - target_coverage=target_coverage_90, - freq="M", - ) - - tuning_rows.append( - { - "partition": str( - partition_meta_candidate.get( - "partition", partition_candidate - ) - ), - "partition_probability_source": partition_probability_source, - "n_score_bins": int(n_score_bins), - "fallback_mode": str( - partition_meta_candidate.get( - "fallback_mode", fallback_mode - ) - ), - "fallback_groups_n": len( - partition_meta_candidate.get("fallback_groups", []) - ), - "alpha_target_90": alpha_target_90, - "alpha_used_90": alpha_used, - "scaled_scores": bool(scaled_scores), - "score_scale_family": str(score_scale_family), - "min_group_size": int(min_group_size), - "empirical_coverage": float( - metrics["empirical_coverage"] - ), - "target_coverage": float(metrics["target_coverage"]), - "coverage_gap": float(metrics["coverage_gap"]), - "avg_interval_width": float( - metrics["avg_interval_width"] - ), - "median_interval_width": float( - metrics["median_interval_width"] - ), - "min_group_coverage": float( - g_metrics["coverage"].min() - ), - "max_group_coverage": float( - g_metrics["coverage"].max() - ), - "std_group_coverage": float( - g_metrics["coverage"].std(ddof=0) - ), - "winkler_90": float( - mean_winkler_score( - y_tune.to_numpy(dtype=float), - y_int, - alpha=alpha_target_90, - ) - ), - "max_monthly_gap": float( - temporal_metrics["max_monthly_gap"] - ), - "stability_over_time": float( - temporal_metrics["stability_over_time"] - ), - } - ) - tuning_completed_candidates += 1 - if ( - tuning_completed_candidates % 1000 == 0 - or tuning_completed_candidates == tuning_total_candidates - ): - _write_tuning_runtime_status( - tuning_status_path, - artifact_namespace=artifact_namespace, - phase="tuning_running", - completed=tuning_completed_candidates, - total=tuning_total_candidates, - started_monotonic=tuning_started, - extra={ - "latest_partition": str(partition_candidate), - "latest_partition_probability_source": str( - partition_probability_source - ), - "latest_n_score_bins": int(n_score_bins), - "latest_fallback_mode": str(fallback_mode), - "latest_alpha_used_90": float(alpha_used), - "latest_score_scale_family": str( - score_scale_family - ), - "latest_min_group_size": int(min_group_size), - }, - ) + if completed % 1000 == 0 or completed == total: + _write_tuning_runtime_status( + status_path, + artifact_namespace=artifact_namespace, + phase="tuning_running", + completed=completed, + total=total, + started_monotonic=started, + extra={ + "latest_partition": str(partition_candidate), + "latest_partition_probability_source": str(partition_probability_source), + "latest_n_score_bins": int(n_score_bins), + "latest_fallback_mode": str(fallback_mode), + "latest_alpha_used_90": float(alpha_used), + "latest_score_scale_family": str(score_scale_family), + "latest_min_group_size": int(min_group_size), + }, + ) _write_tuning_runtime_status( - tuning_status_path, + status_path, artifact_namespace=artifact_namespace, phase="tuning_complete", - completed=tuning_completed_candidates, - total=tuning_total_candidates, - started_monotonic=tuning_started, + completed=len(tuning_rows), + total=total, + started_monotonic=started, ) + return ConformalTuningSearch(tuning_rows=tuning_rows, partition_cache=partition_cache) - tuning_selection = _select_best_tuning_config( - tuning_rows, - partition_candidates=partition_candidates, - alpha_target_90=alpha_target_90, - min_group_coverage_target=min_group_coverage_target, - group_coverage_floor_target_90=group_coverage_floor_target_90, - coverage_guardband_90=coverage_guardband_90, - min_group_guardband_90=min_group_guardband_90, - max_width_budget_90=max_width_budget_90, - target_coverage_90=target_coverage_90, + +def _apply_global_rebalance( + *, + enabled: bool, + min_factor: float, + max_factor: float, + step: float, + y_int_tune_working: np.ndarray, + y_pred_tune: np.ndarray, + y_tune: pd.Series, + y_int_90: np.ndarray, + y_pred_90: np.ndarray, + y_eval_90: pd.Series, + group_tune: pd.Series, + eval_groups_90: pd.Series, + alpha_target_90: float, + target_coverage_90: float, + min_group_coverage_target: float, + metrics_90: dict[str, Any], + group_metrics_90: pd.DataFrame, +) -> GlobalRebalanceResult: + factor = 1.0 + diagnostics: dict[str, float | bool] = { + "enabled": bool(enabled), + "applied": False, + } + if not enabled or len(y_int_tune_working) == 0: + return GlobalRebalanceResult( + y_intervals=y_int_90, + metrics=metrics_90, + group_metrics=group_metrics_90, + factor=factor, + diagnostics=diagnostics, + ) + + min_factor = max(0.05, float(min_factor)) + max_factor = max(min_factor, float(max_factor)) + step = max(0.001, float(step)) + n_steps = int(round((max_factor - min_factor) / step)) + 1 + candidate_factors = np.linspace(min_factor, max_factor, max(2, n_steps)) + + best_trial: dict[str, float] | None = None + tune_y_true = y_tune.to_numpy(dtype=float) + for candidate_factor in candidate_factors: + tune_trial = _scale_intervals_around_prediction( + y_pred_tune, + y_int_tune_working, + float(candidate_factor), + ) + coverage = empirical_interval_coverage(tune_y_true, tune_trial) + min_group_coverage = min_group_interval_coverage(tune_y_true, tune_trial, group_tune) + floor_shortfall = max(0.0, float(min_group_coverage_target) - min_group_coverage) + score = abs(coverage - target_coverage_90) + 100.0 * floor_shortfall + trial = { + "factor": float(candidate_factor), + "coverage": float(coverage), + "min_group_coverage": float(min_group_coverage), + "score": float(score), + } + if best_trial is None or trial["score"] < best_trial["score"]: + best_trial = trial + + if best_trial is None: + return GlobalRebalanceResult( + y_intervals=y_int_90, + metrics=metrics_90, + group_metrics=group_metrics_90, + factor=factor, + diagnostics=diagnostics, + ) + + factor = float(best_trial["factor"]) + diagnostics.update( + { + "factor": factor, + "tune_coverage_after_rebalance": float(best_trial["coverage"]), + "tune_min_group_coverage_after_rebalance": float(best_trial["min_group_coverage"]), + "target_coverage_90": float(target_coverage_90), + "min_group_floor_target": float(min_group_coverage_target), + "applied": abs(factor - 1.0) > 1e-9, + } ) - tuning_df = tuning_selection.tuning_df - best_row = tuning_selection.best_row - selection_tier = tuning_selection.selection_tier - best_cfg = tuning_selection.best_cfg + if abs(factor - 1.0) <= 1e-9: + return GlobalRebalanceResult( + y_intervals=y_int_90, + metrics=metrics_90, + group_metrics=group_metrics_90, + factor=factor, + diagnostics=diagnostics, + ) + logger.info( - "Best 90% tuning config: " - f"partition={best_cfg['partition']}, " - f"prob_source={best_cfg['partition_probability_source']}, " - f"n_bins={best_cfg['n_score_bins']}, " - f"alpha_used={best_cfg['alpha_used_90']}, scaled_scores={best_cfg['scaled_scores']}, " - f"score_scale_family={best_cfg['score_scale_family']}, " - f"min_group_size={best_cfg['min_group_size']}, " - f"coverage={best_row['empirical_coverage']:.4f}, " - f"min_group_coverage={best_row['min_group_coverage']:.4f}, " - f"width={best_row['avg_interval_width']:.4f}, " - f"tier={selection_tier}" + "Applying global interval rebalance factor learned on holdout: " + f"factor={factor:.4f}, " + f"tune_cov={best_trial['coverage']:.4f}, " + f"tune_min_group_cov={best_trial['min_group_coverage']:.4f}" + ) + y_intervals = _scale_intervals_around_prediction(y_pred_90, y_int_90, factor) + metrics = validate_coverage(y_eval_90.to_numpy(dtype=float), y_intervals, alpha_target_90) + group_metrics = conditional_coverage_by_group( + y_eval_90.to_numpy(dtype=float), + y_intervals, + eval_groups_90, + ) + return GlobalRebalanceResult( + y_intervals=y_intervals, + metrics=metrics, + group_metrics=group_metrics, + factor=factor, + diagnostics=diagnostics, ) - group_cal_fit, group_test, partition_meta = _cached_partition_labels( - eval_scope="test", - partition_candidate=best_cfg["partition"], - partition_probability_source=best_cfg["partition_probability_source"], - n_score_bins=best_cfg["n_score_bins"], - fallback_mode=best_cfg["fallback_mode"], - min_group_size=best_cfg["min_group_size"], + +def _select_alpha_95( + *, + alpha_95: float, + alpha_candidates_95: tuple[float, ...], + interval_fit_pred: np.ndarray, + interval_tune_pred: np.ndarray, + y_cal_fit: pd.Series, + y_tune: pd.Series, + group_cal_fit_holdout: pd.Series, + group_tune: pd.Series, + best_cfg: dict[str, Any], +) -> float: + best_alpha = float(alpha_95) + if not alpha_candidates_95: + return best_alpha + best_score: tuple[float, float] | None = None + for alpha_candidate in alpha_candidates_95: + _y_pred, y_int, _diag = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=interval_fit_pred, + y_test_pred=interval_tune_pred, + y_cal=y_cal_fit, + group_cal=group_cal_fit_holdout, + group_test=group_tune, + alpha=float(alpha_candidate), + min_group_size=best_cfg["min_group_size"], + scaled_scores=best_cfg["scaled_scores"], + score_scale_family=best_cfg["score_scale_family"], + log_summary=False, + ) + metrics = validate_coverage( + y_tune.to_numpy(dtype=float), + y_int, + alpha=float(alpha_candidate), + log_summary=False, + ) + score = ( + abs(float(metrics["coverage_gap"])), + float(metrics["avg_interval_width"]), + ) + if best_score is None or score < best_score: + best_score = score + best_alpha = float(alpha_candidate) + return best_alpha + + +def _coverage_metrics( + *, + y_true: pd.Series, + y_intervals: np.ndarray, + groups: pd.Series, + alpha: float, +) -> tuple[dict[str, Any], pd.DataFrame]: + metrics = validate_coverage(y_true.to_numpy(dtype=float), y_intervals, alpha) + group_metrics = conditional_coverage_by_group( + y_true.to_numpy(dtype=float), + y_intervals, + groups, ) - group_cal_fit_holdout, group_tune, _ = _cached_partition_labels( - eval_scope="tune", - partition_candidate=best_cfg["partition"], - partition_probability_source=best_cfg["partition_probability_source"], - n_score_bins=best_cfg["n_score_bins"], - fallback_mode=best_cfg["fallback_mode"], - min_group_size=best_cfg["min_group_size"], + return metrics, group_metrics + + +def _append_width_attr_row( + rows: list[dict[str, Any]], + *, + dataset_scope: str, + stage: str, + y_true: pd.Series, + y_pred: np.ndarray, + y_intervals: np.ndarray, + groups: pd.Series, + issue_dates: pd.Series, + alpha: float, + target_coverage: float, +) -> None: + rows.append( + _stage_metrics( + dataset_scope=dataset_scope, + stage=stage, + y_true=y_true.to_numpy(dtype=float), + y_pred=y_pred, + y_intervals=y_intervals, + groups=groups, + issue_dates=issue_dates, + alpha=alpha, + target_coverage=target_coverage, + ) + ) + + +def _can_use_temporal_segments( + *, + enabled: bool, + issue_tune: pd.Series, + eval_issue: pd.Series, + group_tune: pd.Series, + eval_groups: pd.Series, +) -> bool: + return ( + enabled + and issue_tune.notna().any() + and eval_issue.notna().any() + and len(issue_tune) == len(group_tune) + and len(eval_issue) == len(eval_groups) ) - # Final 90% intervals with tuned config. - y_pred_90, y_int_90, diag_90 = create_pd_intervals_mondrian_from_predictions( + +def _apply_learned_floor_policy( + *, + y_pred: np.ndarray, + y_intervals: np.ndarray, + groups: pd.Series, + group_multipliers: dict[str, float], + temporal_segments: pd.Series | None, + temporal_segment_multipliers: dict[str, float], + global_rebalance_factor: float = 1.0, +) -> np.ndarray: + adjusted = np.asarray(y_intervals, dtype=float).copy() + if group_multipliers: + adjusted = apply_group_multipliers(y_pred, adjusted, groups, group_multipliers) + if temporal_segment_multipliers and temporal_segments is not None: + adjusted = apply_group_multipliers( + y_pred, + adjusted, + temporal_segments, + temporal_segment_multipliers, + ) + if abs(global_rebalance_factor - 1.0) > 1e-9: + adjusted = _scale_intervals_around_prediction(y_pred, adjusted, global_rebalance_factor) + return adjusted + + +def _build_90_interval_evidence( + *, + evaluation_scope_key: str, + y_test: pd.Series, + y_tune: pd.Series, + issue_test: pd.Series, + issue_tune: pd.Series, + interval_fit_pred: np.ndarray, + interval_test_pred: np.ndarray, + interval_tune_pred: np.ndarray, + y_cal_fit: pd.Series, + group_cal_fit: pd.Series, + group_test: pd.Series, + group_cal_fit_holdout: pd.Series, + group_tune: pd.Series, + best_cfg: dict[str, Any], + alpha_target_90: float, + target_coverage_90: float, + group_coverage_floor_enabled: bool, + group_coverage_floor_target_90: float, + group_multiplier_grid: tuple[float, ...], + temporal_segment_floor_enabled: bool, + temporal_segment_freq: str, + temporal_segment_min_size: int, + temporal_multiplier_grid: tuple[float, ...], + shrinkback_enabled: bool, + min_group_coverage_target: float, + global_rebalance_enabled: bool, + global_rebalance_min_factor: float, + global_rebalance_max_factor: float, + global_rebalance_step: float, +) -> ConformalEvidence90: + eval_scope = "test" if evaluation_scope_key == "test" else "holdout" + y_eval = y_test if evaluation_scope_key == "test" else y_tune + eval_groups = group_test if evaluation_scope_key == "test" else group_tune + eval_issue = issue_test if evaluation_scope_key == "test" else issue_tune + + y_pred, y_intervals, diag = create_pd_intervals_mondrian_from_predictions( y_cal_pred=interval_fit_pred, y_test_pred=interval_test_pred if evaluation_scope_key == "test" else interval_tune_pred, y_cal=y_cal_fit, group_cal=group_cal_fit, - group_test=group_test if evaluation_scope_key == "test" else group_tune, + group_test=eval_groups, alpha=best_cfg["alpha_used_90"], min_group_size=best_cfg["min_group_size"], scaled_scores=best_cfg["scaled_scores"], score_scale_family=best_cfg["score_scale_family"], ) - y_eval_90 = y_test if evaluation_scope_key == "test" else y_tune - eval_groups_90 = group_test if evaluation_scope_key == "test" else group_tune - eval_issue_90 = issue_test if evaluation_scope_key == "test" else issue_tune - metrics_90 = validate_coverage(y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 + metrics, group_metrics = _coverage_metrics( + y_true=y_eval, + y_intervals=y_intervals, + groups=eval_groups, + alpha=alpha_target_90, ) - width_attr_rows: list[dict[str, Any]] = [ - _stage_metrics( - dataset_scope="test" if evaluation_scope_key == "test" else "holdout", - stage="base_interval", - y_true=y_eval_90.to_numpy(dtype=float), - y_pred=y_pred_90, - y_intervals=y_int_90, - groups=eval_groups_90, - issue_dates=eval_issue_90, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) - ] - # Learn group multipliers on calibration holdout only (no test-label adaptation). + width_attr_rows: list[dict[str, Any]] = [] + _append_width_attr_row( + width_attr_rows, + dataset_scope=eval_scope, + stage="base_interval", + y_true=y_eval, + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + issue_dates=eval_issue, + alpha=alpha_target_90, + target_coverage=target_coverage_90, + ) + y_pred_tune, y_int_tune, _diag_tune = create_pd_intervals_mondrian_from_predictions( y_cal_pred=interval_fit_pred, y_test_pred=interval_tune_pred, @@ -1203,108 +1516,101 @@ def _cached_partition_labels( score_scale_family=best_cfg["score_scale_family"], log_summary=False, ) - tune_metrics_90_before = validate_coverage( + tune_metrics_before_floor = validate_coverage( y_tune.to_numpy(dtype=float), y_int_tune, alpha_target_90 ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="tune_holdout", - stage="base_interval", - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_tune, - groups=group_tune, - issue_dates=issue_tune, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope="tune_holdout", + stage="base_interval", + y_true=y_tune, + y_pred=y_pred_tune, + y_intervals=y_int_tune, + groups=group_tune, + issue_dates=issue_tune, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) + if group_coverage_floor_enabled: - y_int_90_adjusted, group_multipliers, coverage_floor_report = enforce_group_coverage_floor( - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_tune, - groups=group_tune, - target_coverage=group_coverage_floor_target_90, - multiplier_grid=group_multiplier_grid, + y_int_tune_after_group, group_multipliers, coverage_floor_report = ( + enforce_group_coverage_floor( + y_true=y_tune.to_numpy(dtype=float), + y_pred=y_pred_tune, + y_intervals=y_int_tune, + groups=group_tune, + target_coverage=group_coverage_floor_target_90, + multiplier_grid=group_multiplier_grid, + ) ) else: - y_int_90_adjusted = np.asarray(y_int_tune, dtype=float).copy() + y_int_tune_after_group = np.asarray(y_int_tune, dtype=float).copy() group_multipliers = {} - coverage_floor_report = pd.DataFrame( - columns=[ - "group", - "coverage_before", - "coverage_after", - "target_coverage", - "multiplier", - "adjusted", - ] - ) - tune_metrics_90_after = validate_coverage( - y_tune.to_numpy(dtype=float), y_int_90_adjusted, alpha_target_90 + coverage_floor_report = _empty_coverage_floor_report() + tune_metrics_after_floor = validate_coverage( + y_tune.to_numpy(dtype=float), y_int_tune_after_group, alpha_target_90 ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="tune_holdout", - stage="after_group_floor", - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_90_adjusted, - groups=group_tune, - issue_dates=issue_tune, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope="tune_holdout", + stage="after_group_floor", + y_true=y_tune, + y_pred=y_pred_tune, + y_intervals=y_int_tune_after_group, + groups=group_tune, + issue_dates=issue_tune, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) + eval_temporal_segments: pd.Series | None = None + tune_temporal_segments: pd.Series | None = None temporal_segment_multipliers: dict[str, float] = {} - temporal_segment_report = pd.DataFrame( - columns=[ - "segment", - "support_n", - "coverage_before", - "coverage_after", - "target_coverage", - "min_segment_size", - "multiplier", - "adjusted", - ] - ) - y_int_90_tune_working = y_int_90_adjusted - tune_metrics_90_after_temporal = tune_metrics_90_after.copy() - y_int_90_base_test = np.asarray(y_int_90, dtype=float).copy() + temporal_segment_report = _empty_temporal_segment_report() + y_int_tune_working = y_int_tune_after_group + tune_metrics_after_temporal = tune_metrics_after_floor.copy() + y_intervals_base_eval = np.asarray(y_intervals, dtype=float).copy() + if group_multipliers: logger.info( "Applying group coverage floor multipliers learned on calibration holdout: " f"{group_multipliers}" ) - y_int_90 = apply_group_multipliers(y_pred_90, y_int_90, eval_groups_90, group_multipliers) - metrics_90 = validate_coverage(y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 + y_intervals = _apply_learned_floor_policy( + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + group_multipliers=group_multipliers, + temporal_segments=None, + temporal_segment_multipliers={}, ) - else: - logger.info("No group coverage floor adjustments were required.") - width_attr_rows.append( - _stage_metrics( - dataset_scope="test" if evaluation_scope_key == "test" else "holdout", - stage="after_group_floor", - y_true=y_eval_90.to_numpy(dtype=float), - y_pred=y_pred_90, - y_intervals=y_int_90, - groups=eval_groups_90, - issue_dates=eval_issue_90, + metrics, group_metrics = _coverage_metrics( + y_true=y_eval, + y_intervals=y_intervals, + groups=eval_groups, alpha=alpha_target_90, - target_coverage=target_coverage_90, ) + else: + logger.info("No group coverage floor adjustments were required.") + _append_width_attr_row( + width_attr_rows, + dataset_scope=eval_scope, + stage="after_group_floor", + y_true=y_eval, + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + issue_dates=eval_issue, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) - if ( - temporal_segment_floor_enabled - and issue_tune.notna().any() - and eval_issue_90.notna().any() - and len(issue_tune) == len(group_tune) - and len(eval_issue_90) == len(eval_groups_90) + + if _can_use_temporal_segments( + enabled=temporal_segment_floor_enabled, + issue_tune=issue_tune, + eval_issue=eval_issue, + group_tune=group_tune, + eval_groups=eval_groups, ): tune_temporal_segments = build_group_temporal_segments( groups=group_tune, @@ -1312,98 +1618,84 @@ def _cached_partition_labels( freq=temporal_segment_freq, ) eval_temporal_segments = build_group_temporal_segments( - groups=eval_groups_90, - issue_dates=eval_issue_90, + groups=eval_groups, + issue_dates=eval_issue, freq=temporal_segment_freq, ) - y_int_90_tune_temporal, temporal_segment_multipliers, temporal_segment_report = ( + y_int_tune_temporal, temporal_segment_multipliers, temporal_segment_report = ( enforce_segment_coverage_floor( y_true=y_tune.to_numpy(dtype=float), y_pred=y_pred_tune, - y_intervals=y_int_90_tune_working, + y_intervals=y_int_tune_working, segments=tune_temporal_segments, target_coverage=group_coverage_floor_target_90, min_segment_size=temporal_segment_min_size, multiplier_grid=temporal_multiplier_grid, ) ) - y_int_90_tune_working = y_int_90_tune_temporal - tune_metrics_90_after_temporal = validate_coverage( - y_tune.to_numpy(dtype=float), y_int_90_tune_temporal, alpha_target_90 + y_int_tune_working = y_int_tune_temporal + tune_metrics_after_temporal = validate_coverage( + y_tune.to_numpy(dtype=float), y_int_tune_temporal, alpha_target_90 ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="tune_holdout", - stage="after_temporal_floor", - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_90_tune_temporal, - groups=group_tune, - issue_dates=issue_tune, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope="tune_holdout", + stage="after_temporal_floor", + y_true=y_tune, + y_pred=y_pred_tune, + y_intervals=y_int_tune_temporal, + groups=group_tune, + issue_dates=issue_tune, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) if temporal_segment_multipliers: logger.info( "Applying temporal coverage floor multipliers learned on holdout " f"(freq={temporal_segment_freq}): {temporal_segment_multipliers}" ) - y_int_90 = apply_group_multipliers( - y_pred_90, - y_int_90, - eval_temporal_segments, - temporal_segment_multipliers, - ) - metrics_90 = validate_coverage( - y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90 + y_intervals = _apply_learned_floor_policy( + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + group_multipliers={}, + temporal_segments=eval_temporal_segments, + temporal_segment_multipliers=temporal_segment_multipliers, ) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 + metrics, group_metrics = _coverage_metrics( + y_true=y_eval, + y_intervals=y_intervals, + groups=eval_groups, + alpha=alpha_target_90, ) else: logger.info("No temporal segment coverage adjustments were required.") elif temporal_segment_floor_enabled: logger.info("Temporal segment coverage adjustments skipped (missing issue_d coverage).") - width_attr_rows.append( - _stage_metrics( - dataset_scope="test" if evaluation_scope_key == "test" else "holdout", - stage="after_temporal_floor", - y_true=y_eval_90.to_numpy(dtype=float), - y_pred=y_pred_90, - y_intervals=y_int_90, - groups=eval_groups_90, - issue_dates=eval_issue_90, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope=eval_scope, + stage="after_temporal_floor", + y_true=y_eval, + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + issue_dates=eval_issue, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) - shrinkback_report = pd.DataFrame( - columns=[ - "stage", - "factor_scope", - "factor_key", - "candidate_factor", - "accepted", - "coverage", - "min_group_coverage", - "avg_width", - "winkler_90", - "max_monthly_gap", - "stability_over_time", - ] - ) + shrinkback_report = _empty_shrinkback_report() if shrinkback_enabled and (group_multipliers or temporal_segment_multipliers): shrink_max_monthly_gap = temporal_stability_summary( y_tune.to_numpy(dtype=float), - y_int_90_tune_working, + y_int_tune_working, issue_tune, target_coverage=target_coverage_90, freq="M", )["max_monthly_gap"] ( - y_int_90_tune_working, + y_int_tune_working, group_multipliers, temporal_segment_multipliers, shrinkback_report, @@ -1414,7 +1706,7 @@ def _cached_partition_labels( groups=group_tune, issue_dates=issue_tune, group_factors=group_multipliers, - temporal_segments=tune_temporal_segments if temporal_segment_floor_enabled else None, + temporal_segments=tune_temporal_segments, temporal_factors=temporal_segment_multipliers, target_coverage=target_coverage_90, min_group_coverage_target=min_group_coverage_target, @@ -1425,150 +1717,105 @@ def _cached_partition_labels( group_multiplier_grid=group_multiplier_grid, temporal_multiplier_grid=temporal_multiplier_grid, ) - y_int_90 = np.asarray(y_int_90_base_test, dtype=float).copy() - if group_multipliers: - y_int_90 = apply_group_multipliers( - y_pred_90, y_int_90, eval_groups_90, group_multipliers - ) - if temporal_segment_multipliers and eval_temporal_segments is not None: - y_int_90 = apply_group_multipliers( - y_pred_90, - y_int_90, - eval_temporal_segments, - temporal_segment_multipliers, - ) - metrics_90 = validate_coverage(y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 + y_intervals = _apply_learned_floor_policy( + y_pred=y_pred, + y_intervals=y_intervals_base_eval, + groups=eval_groups, + group_multipliers=group_multipliers, + temporal_segments=eval_temporal_segments, + temporal_segment_multipliers=temporal_segment_multipliers, ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="tune_holdout", - stage="after_shrinkback", - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_90_tune_working, - groups=group_tune, - issue_dates=issue_tune, + metrics, group_metrics = _coverage_metrics( + y_true=y_eval, + y_intervals=y_intervals, + groups=eval_groups, alpha=alpha_target_90, - target_coverage=target_coverage_90, ) + _append_width_attr_row( + width_attr_rows, + dataset_scope="tune_holdout", + stage="after_shrinkback", + y_true=y_tune, + y_pred=y_pred_tune, + y_intervals=y_int_tune_working, + groups=group_tune, + issue_dates=issue_tune, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="test" if evaluation_scope_key == "test" else "holdout", - stage="after_shrinkback", - y_true=y_eval_90.to_numpy(dtype=float), - y_pred=y_pred_90, - y_intervals=y_int_90, - groups=eval_groups_90, - issue_dates=eval_issue_90, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope=eval_scope, + stage="after_shrinkback", + y_true=y_eval, + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + issue_dates=eval_issue, + alpha=alpha_target_90, + target_coverage=target_coverage_90, + ) + + rebalance_result = _apply_global_rebalance( + enabled=global_rebalance_enabled, + min_factor=global_rebalance_min_factor, + max_factor=global_rebalance_max_factor, + step=global_rebalance_step, + y_int_tune_working=y_int_tune_working, + y_pred_tune=y_pred_tune, + y_tune=y_tune, + y_int_90=y_intervals, + y_pred_90=y_pred, + y_eval_90=y_eval, + group_tune=group_tune, + eval_groups_90=eval_groups, + alpha_target_90=alpha_target_90, + target_coverage_90=target_coverage_90, + min_group_coverage_target=min_group_coverage_target, + metrics_90=metrics, + group_metrics_90=group_metrics, + ) + return ConformalEvidence90( + y_eval=y_eval, + eval_groups=eval_groups, + eval_issue=eval_issue, + y_pred=y_pred, + y_intervals=rebalance_result.y_intervals, + diag=diag, + metrics=rebalance_result.metrics, + group_metrics=rebalance_result.group_metrics, + y_pred_tune=y_pred_tune, + y_intervals_tune_base=y_int_tune, + tune_metrics_before_floor=tune_metrics_before_floor, + tune_metrics_after_floor=tune_metrics_after_floor, + tune_metrics_after_temporal_floor=tune_metrics_after_temporal, + group_multipliers=group_multipliers, + coverage_floor_report=coverage_floor_report, + temporal_segment_multipliers=temporal_segment_multipliers, + temporal_segment_report=temporal_segment_report, + shrinkback_report=shrinkback_report, + eval_temporal_segments=eval_temporal_segments, + width_attr_rows=width_attr_rows, + global_rebalance_factor=rebalance_result.factor, + global_rebalance_diagnostics=rebalance_result.diagnostics, ) - # Optional global rebalance: tune one uniform radius factor on calibration holdout - # to get closer to nominal global coverage while preserving minimum group floor. - global_rebalance_factor = 1.0 - global_rebalance_diagnostics: dict[str, float | bool] = { - "enabled": bool(global_rebalance_enabled), - "applied": False, - } - if global_rebalance_enabled and len(y_int_90_tune_working) > 0: - min_factor = max(0.05, float(global_rebalance_min_factor)) - max_factor = max(min_factor, float(global_rebalance_max_factor)) - step = max(0.001, float(global_rebalance_step)) - n_steps = int(round((max_factor - min_factor) / step)) + 1 - candidate_factors = np.linspace(min_factor, max_factor, max(2, n_steps)) - - tune_y_true = y_tune.to_numpy(dtype=float) - tune_target_cov = target_coverage_90 - tune_group_floor = float(min_group_coverage_target) - - best_trial: dict[str, float] | None = None - for factor in candidate_factors: - tune_trial = _scale_intervals_around_prediction( - y_pred_tune, y_int_90_tune_working, factor - ) - cov_trial = empirical_interval_coverage(tune_y_true, tune_trial) - min_group_cov_trial = min_group_interval_coverage(tune_y_true, tune_trial, group_tune) - floor_shortfall = max(0.0, tune_group_floor - min_group_cov_trial) - score = abs(cov_trial - tune_target_cov) + 100.0 * floor_shortfall - trial = { - "factor": float(factor), - "coverage": float(cov_trial), - "min_group_coverage": float(min_group_cov_trial), - "score": float(score), - } - if best_trial is None or trial["score"] < best_trial["score"]: - best_trial = trial - - if best_trial is not None: - global_rebalance_factor = float(best_trial["factor"]) - global_rebalance_diagnostics.update( - { - "factor": global_rebalance_factor, - "tune_coverage_after_rebalance": float(best_trial["coverage"]), - "tune_min_group_coverage_after_rebalance": float( - best_trial["min_group_coverage"] - ), - "target_coverage_90": float(tune_target_cov), - "min_group_floor_target": float(tune_group_floor), - "applied": abs(global_rebalance_factor - 1.0) > 1e-9, - } - ) - if abs(global_rebalance_factor - 1.0) > 1e-9: - logger.info( - "Applying global interval rebalance factor learned on holdout: " - f"factor={global_rebalance_factor:.4f}, " - f"tune_cov={best_trial['coverage']:.4f}, " - f"tune_min_group_cov={best_trial['min_group_coverage']:.4f}" - ) - y_int_90 = _scale_intervals_around_prediction( - y_pred_90, y_int_90, global_rebalance_factor - ) - metrics_90 = validate_coverage( - y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90 - ) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 - ) - - best_alpha_95 = float(alpha_95) - if alpha_candidates_95: - best_score_95: tuple[float, float] | None = None - for alpha_candidate_95 in alpha_candidates_95: - _y_pred_95_tune, y_int_95_tune, _diag_95_tune = ( - create_pd_intervals_mondrian_from_predictions( - y_cal_pred=interval_fit_pred, - y_test_pred=interval_tune_pred, - y_cal=y_cal_fit, - group_cal=group_cal_fit_holdout, - group_test=group_tune, - alpha=float(alpha_candidate_95), - min_group_size=best_cfg["min_group_size"], - scaled_scores=best_cfg["scaled_scores"], - score_scale_family=best_cfg["score_scale_family"], - log_summary=False, - ) - ) - metrics_95_tune = validate_coverage( - y_tune.to_numpy(dtype=float), - y_int_95_tune, - alpha=float(alpha_candidate_95), - log_summary=False, - ) - score_95 = ( - abs(float(metrics_95_tune["coverage_gap"])), - float(metrics_95_tune["avg_interval_width"]), - ) - if best_score_95 is None or score_95 < best_score_95: - best_score_95 = score_95 - best_alpha_95 = float(alpha_candidate_95) - # 95% intervals using same structure settings for consistency. - y_pred_95, y_int_95, diag_95 = create_pd_intervals_mondrian_from_predictions( +def _build_95_interval_evidence( + *, + evaluation_scope_key: str, + interval_fit_pred: np.ndarray, + interval_test_pred: np.ndarray, + interval_tune_pred: np.ndarray, + y_cal_fit: pd.Series, + group_cal_fit: pd.Series, + group_test: pd.Series, + group_tune: pd.Series, + evidence_90: ConformalEvidence90, + best_cfg: dict[str, Any], + best_alpha_95: float, +) -> ConformalEvidence95: + y_pred, y_intervals, diag = create_pd_intervals_mondrian_from_predictions( y_cal_pred=interval_fit_pred, y_test_pred=interval_test_pred if evaluation_scope_key == "test" else interval_tune_pred, y_cal=y_cal_fit, @@ -1579,50 +1826,107 @@ def _cached_partition_labels( scaled_scores=best_cfg["scaled_scores"], score_scale_family=best_cfg["score_scale_family"], ) - if group_multipliers: - y_int_95 = apply_group_multipliers(y_pred_95, y_int_95, eval_groups_90, group_multipliers) - if temporal_segment_multipliers and eval_temporal_segments is not None: - y_int_95 = apply_group_multipliers( - y_pred_95, y_int_95, eval_temporal_segments, temporal_segment_multipliers - ) - if abs(global_rebalance_factor - 1.0) > 1e-9: - y_int_95 = _scale_intervals_around_prediction(y_pred_95, y_int_95, global_rebalance_factor) - metrics_95 = validate_coverage(y_eval_90.to_numpy(dtype=float), y_int_95, best_alpha_95) - group_metrics_95 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_95, eval_groups_90 + y_intervals = _apply_learned_floor_policy( + y_pred=y_pred, + y_intervals=y_intervals, + groups=evidence_90.eval_groups, + group_multipliers=evidence_90.group_multipliers, + temporal_segments=evidence_90.eval_temporal_segments, + temporal_segment_multipliers=evidence_90.temporal_segment_multipliers, + global_rebalance_factor=evidence_90.global_rebalance_factor, + ) + metrics, group_metrics = _coverage_metrics( + y_true=evidence_90.y_eval, + y_intervals=y_intervals, + groups=evidence_90.eval_groups, + alpha=best_alpha_95, + ) + return ConformalEvidence95( + y_pred=y_pred, + y_intervals=y_intervals, + diag=diag, + metrics=metrics, + group_metrics=group_metrics, ) - # Compose output tables. - intervals_payload = { - "y_true": y_eval_90.to_numpy(dtype=float), - "y_pred": y_pred_90, - "pd_low_90": y_int_90[:, 0], - "pd_high_90": y_int_90[:, 1], - "pd_low_95": y_int_95[:, 0], - "pd_high_95": y_int_95[:, 1], - "width_90": y_int_90[:, 1] - y_int_90[:, 0], - "width_95": y_int_95[:, 1] - y_int_95[:, 0], - GROUP_COL: eval_groups_90.to_numpy(dtype=str), - "loan_amnt": ( - test_df["loan_amnt"].to_numpy(dtype=float) - if evaluation_scope_key == "test" and "loan_amnt" in test_df.columns - else cal_df.iloc[idx_cal_tune]["loan_amnt"].reset_index(drop=True).to_numpy(dtype=float) - if evaluation_scope_key == "holdout" and "loan_amnt" in cal_df.columns - else np.nan + +def _eval_source_frame( + *, + evaluation_scope_key: str, + test_df: pd.DataFrame, + cal_df: pd.DataFrame, + idx_cal_tune: np.ndarray, +) -> pd.DataFrame: + if evaluation_scope_key == "test": + return test_df.reset_index(drop=True) + return cal_df.iloc[idx_cal_tune].reset_index(drop=True) + + +def _eval_loan_amount_values( + *, + evaluation_scope_key: str, + test_df: pd.DataFrame, + cal_df: pd.DataFrame, + idx_cal_tune: np.ndarray, +) -> np.ndarray | float: + if evaluation_scope_key == "test" and "loan_amnt" in test_df.columns: + return test_df["loan_amnt"].to_numpy(dtype=float) + if evaluation_scope_key == "holdout" and "loan_amnt" in cal_df.columns: + return cal_df.iloc[idx_cal_tune]["loan_amnt"].reset_index(drop=True).to_numpy(dtype=float) + return np.nan + + +def _build_intervals_table( + *, + y_eval_90: pd.Series, + y_pred_90: np.ndarray, + y_int_90: np.ndarray, + y_int_95: np.ndarray, + eval_groups_90: pd.Series, + eval_temporal_segments: pd.Series | None, + evaluation_scope_key: str, + test_df: pd.DataFrame, + cal_df: pd.DataFrame, + idx_cal_tune: np.ndarray, +) -> pd.DataFrame: + eval_df = _eval_source_frame( + evaluation_scope_key=evaluation_scope_key, + test_df=test_df, + cal_df=cal_df, + idx_cal_tune=idx_cal_tune, + ) + intervals_payload: dict[str, Any] = { + "y_true": y_eval_90.to_numpy(dtype=float), + "y_pred": y_pred_90, + "pd_low_90": y_int_90[:, 0], + "pd_high_90": y_int_90[:, 1], + "pd_low_95": y_int_95[:, 0], + "pd_high_95": y_int_95[:, 1], + "width_90": y_int_90[:, 1] - y_int_90[:, 0], + "width_95": y_int_95[:, 1] - y_int_95[:, 0], + GROUP_COL: eval_groups_90.to_numpy(dtype=str), + "loan_amnt": _eval_loan_amount_values( + evaluation_scope_key=evaluation_scope_key, + test_df=test_df, + cal_df=cal_df, + idx_cal_tune=idx_cal_tune, ), } - eval_df = ( - test_df.reset_index(drop=True) - if evaluation_scope_key == "test" - else cal_df.iloc[idx_cal_tune].reset_index(drop=True) - ) if "id" in eval_df.columns: intervals_payload["id"] = eval_df["id"].astype(str).to_numpy() if eval_temporal_segments is not None: intervals_payload["temporal_segment"] = eval_temporal_segments.to_numpy(dtype=str) intervals_df = pd.DataFrame(intervals_payload) - intervals_df.insert(0, "_row_number", range(len(intervals_df))) + intervals_df.insert(0, "_row_number", np.arange(len(intervals_df), dtype=int)) + return intervals_df + +def _build_group_metrics_table( + *, + group_metrics_90: pd.DataFrame, + group_metrics_95: pd.DataFrame, + coverage_floor_report: pd.DataFrame, +) -> pd.DataFrame: gm90 = group_metrics_90.rename( columns={ "coverage": "coverage_90", @@ -1642,7 +1946,7 @@ def _cached_partition_labels( on="group", how="outer", ).sort_values("group") - group_metrics_df = group_metrics_df.merge( + return group_metrics_df.merge( coverage_floor_report[ ["group", "coverage_before", "coverage_after", "multiplier", "adjusted"] ], @@ -1650,31 +1954,82 @@ def _cached_partition_labels( how="left", ) - # Persist artifacts. - paths = _resolve_artifact_paths(artifact_namespace) - intervals_mondrian_path = paths["intervals"] - group_metrics_path = paths["group_metrics"] - tuning_path = paths["tuning"] - pareto_path = paths["pareto"] - coverage_floor_path = paths["group_floor"] - temporal_coverage_floor_path = paths["temporal_floor"] - shrinkback_path = paths["shrinkback"] - width_attr_path = paths["width_attr"] - results_path = paths["results"] - width_attr_status_path = paths["width_attr_status"] - resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - intervals_df.to_parquet(intervals_mondrian_path, index=False) - group_metrics_df.to_parquet(group_metrics_path, index=False) - tuning_df.to_parquet(tuning_path, index=False) - tuning_df[tuning_df["is_pareto"]].copy().to_parquet(pareto_path, index=False) - coverage_floor_report.to_parquet(coverage_floor_path, index=False) - temporal_segment_report.to_parquet(temporal_coverage_floor_path, index=False) - shrinkback_report.to_parquet(shrinkback_path, index=False) - width_attr_df = pd.DataFrame(width_attr_rows) - width_attr_df.to_parquet(width_attr_path, index=False) - - payload = { +def _build_conformal_artifact_tables( + *, + y_eval_90: pd.Series, + y_pred_90: np.ndarray, + y_int_90: np.ndarray, + y_int_95: np.ndarray, + eval_groups_90: pd.Series, + eval_temporal_segments: pd.Series | None, + evaluation_scope_key: str, + test_df: pd.DataFrame, + cal_df: pd.DataFrame, + idx_cal_tune: np.ndarray, + group_metrics_90: pd.DataFrame, + group_metrics_95: pd.DataFrame, + coverage_floor_report: pd.DataFrame, + width_attr_rows: list[dict[str, Any]], +) -> ConformalArtifactTables: + return ConformalArtifactTables( + intervals=_build_intervals_table( + y_eval_90=y_eval_90, + y_pred_90=y_pred_90, + y_int_90=y_int_90, + y_int_95=y_int_95, + eval_groups_90=eval_groups_90, + eval_temporal_segments=eval_temporal_segments, + evaluation_scope_key=evaluation_scope_key, + test_df=test_df, + cal_df=cal_df, + idx_cal_tune=idx_cal_tune, + ), + group_metrics=_build_group_metrics_table( + group_metrics_90=group_metrics_90, + group_metrics_95=group_metrics_95, + coverage_floor_report=coverage_floor_report, + ), + width_attribution=pd.DataFrame(width_attr_rows), + ) + + +def _build_conformal_results_payload( + *, + model_path: Path, + calibrator_override_path: str | None, + metrics_90: dict[str, Any], + metrics_95: dict[str, Any], + diag_90: dict[str, Any], + diag_95: dict[str, Any], + partition_meta: dict[str, Any], + partition: str, + group_metrics_90: pd.DataFrame, + group_metrics_95: pd.DataFrame, + best_cfg: dict[str, Any], + alpha_candidates_95: tuple[float, ...], + best_alpha_95: float, + paths: dict[str, Path], + group_multipliers: dict[str, float], + temporal_segment_floor_enabled: bool, + temporal_segment_freq: str, + temporal_segment_min_size: int, + temporal_segment_multipliers: dict[str, float], + group_coverage_floor_enabled: bool, + shrinkback_enabled: bool, + global_rebalance_diagnostics: dict[str, float | bool], + group_coverage_floor_target_90: float, + X_cal_fit: pd.DataFrame, + X_tune: pd.DataFrame, + tuning_holdout_ratio: float, + tuning_random_state: int, + calibration_fraction: float, + evaluation_scope_key: str, + tune_metrics_90_before: dict[str, Any], + tune_metrics_90_after: dict[str, Any], + tune_metrics_90_after_temporal: dict[str, Any], +) -> dict[str, Any]: + return { "model_path": str(model_path), "calibrator_override_path": str(calibrator_override_path or ""), "metrics_90": {k: to_python_scalar(v) for k, v in metrics_90.items()}, @@ -1688,21 +2043,21 @@ def _cached_partition_labels( "tuning_90_best": best_cfg, "alpha_candidates_95": [float(x) for x in alpha_candidates_95], "alpha_used_95": float(best_alpha_95), - "tuning_90_table_path": str(tuning_path), - "tuning_90_pareto_path": str(pareto_path), - "group_coverage_floor_path": str(coverage_floor_path), + "tuning_90_table_path": str(paths["tuning"]), + "tuning_90_pareto_path": str(paths["pareto"]), + "group_coverage_floor_path": str(paths["group_floor"]), "group_coverage_multipliers": {k: float(v) for k, v in group_multipliers.items()}, "temporal_segment_floor_enabled": bool(temporal_segment_floor_enabled), "temporal_segment_freq": str(temporal_segment_freq), "temporal_segment_min_size": int(temporal_segment_min_size), - "temporal_segment_coverage_floor_path": str(temporal_coverage_floor_path), + "temporal_segment_coverage_floor_path": str(paths["temporal_floor"]), "temporal_segment_multipliers": { k: float(v) for k, v in temporal_segment_multipliers.items() }, "group_coverage_floor_enabled": bool(group_coverage_floor_enabled), "shrinkback_enabled": bool(shrinkback_enabled), - "shrinkback_path": str(shrinkback_path), - "width_attribution_path": str(width_attr_path), + "shrinkback_path": str(paths["shrinkback"]), + "width_attribution_path": str(paths["width_attr"]), "global_rebalance": global_rebalance_diagnostics, "group_coverage_floor_target_90": float(group_coverage_floor_target_90), "calibration_split": { @@ -1724,9 +2079,24 @@ def _cached_partition_labels( k: to_python_scalar(v) for k, v in tune_metrics_90_after_temporal.items() }, } - with open(results_path, "wb") as f: - pickle.dump(payload, f) - width_attr_status_path.write_text( + + +def _write_width_attribution_status( + *, + status_path: Path, + artifact_namespace: str | None, + partition_meta: dict[str, Any], + partition: str, + best_cfg: dict[str, Any], + best_alpha_95: float, + group_multipliers: dict[str, float], + temporal_segment_multipliers: dict[str, float], + width_attr_path: Path, + shrinkback_path: Path, + evaluation_scope_key: str, + resolved_run_tag: str, +) -> None: + status_path.write_text( json.dumps( { "artifact_namespace": artifact_namespace or "", @@ -1759,14 +2129,21 @@ def _cached_partition_labels( encoding="utf-8", ) + +def _log_conformal_artifacts( + paths: dict[str, Path], metrics_90: dict[str, Any], metrics_95: dict[str, Any] +) -> None: logger.info("Conformal artifacts saved:") - logger.info(f" - {intervals_mondrian_path}") - logger.info(f" - {group_metrics_path}") - logger.info(f" - {tuning_path}") - logger.info(f" - {pareto_path}") - logger.info(f" - {coverage_floor_path}") - logger.info(f" - {temporal_coverage_floor_path}") - logger.info(f" - {results_path}") + for key in [ + "intervals", + "group_metrics", + "tuning", + "pareto", + "group_floor", + "temporal_floor", + "results", + ]: + logger.info(f" - {paths[key]}") logger.info( "Final metrics: " f"90% coverage={metrics_90['empirical_coverage']:.4f} " @@ -1776,6 +2153,425 @@ def _cached_partition_labels( ) +def _persist_conformal_artifacts( + *, + paths: dict[str, Path], + artifact_namespace: str | None, + run_tag: str | None, + tables: ConformalArtifactTables, + tuning_df: pd.DataFrame, + coverage_floor_report: pd.DataFrame, + temporal_segment_report: pd.DataFrame, + shrinkback_report: pd.DataFrame, + payload: dict[str, Any], + partition_meta: dict[str, Any], + partition: str, + best_cfg: dict[str, Any], + best_alpha_95: float, + group_multipliers: dict[str, float], + temporal_segment_multipliers: dict[str, float], + evaluation_scope_key: str, + metrics_90: dict[str, Any], + metrics_95: dict[str, Any], +) -> dict[str, Path]: + tables.intervals.to_parquet(paths["intervals"], index=False) + tables.group_metrics.to_parquet(paths["group_metrics"], index=False) + tuning_df.to_parquet(paths["tuning"], index=False) + tuning_df[tuning_df["is_pareto"]].copy().to_parquet(paths["pareto"], index=False) + coverage_floor_report.to_parquet(paths["group_floor"], index=False) + temporal_segment_report.to_parquet(paths["temporal_floor"], index=False) + shrinkback_report.to_parquet(paths["shrinkback"], index=False) + tables.width_attribution.to_parquet(paths["width_attr"], index=False) + + with open(paths["results"], "wb") as f: + pickle.dump(payload, f) + + resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) + _write_width_attribution_status( + status_path=paths["width_attr_status"], + artifact_namespace=artifact_namespace, + partition_meta=partition_meta, + partition=partition, + best_cfg=best_cfg, + best_alpha_95=best_alpha_95, + group_multipliers=group_multipliers, + temporal_segment_multipliers=temporal_segment_multipliers, + width_attr_path=paths["width_attr"], + shrinkback_path=paths["shrinkback"], + evaluation_scope_key=evaluation_scope_key, + resolved_run_tag=resolved_run_tag, + ) + _log_conformal_artifacts(paths, metrics_90, metrics_95) + return paths + + +def _parse_float_tuple(raw: str) -> tuple[float, ...]: + values = [float(token.strip()) for token in str(raw).split(",") if token.strip()] + if not values: + raise ValueError("Expected at least one float value.") + return tuple(values) + + +def _parse_int_tuple(raw: str) -> tuple[int, ...]: + values = [int(token.strip()) for token in str(raw).split(",") if token.strip()] + if not values: + raise ValueError("Expected at least one integer value.") + return tuple(values) + + +def _parse_bool_tuple(raw: str) -> tuple[bool, ...]: + values = [ + token.strip().lower() in {"1", "true", "yes", "y"} + for token in str(raw).split(",") + if token.strip() + ] + if not values: + raise ValueError("Expected at least one boolean value.") + return tuple(values) + + +def _parse_str_tuple(raw: str) -> tuple[str, ...]: + values = tuple(token.strip() for token in str(raw).split(",") if token.strip()) + if not values: + raise ValueError("Expected at least one string value.") + return values + + +def main( + alpha_target_90: float = 0.10, + alpha_95: float = 0.05, + alpha_candidates_90: tuple[float, ...] = (0.10, 0.095, 0.09, 0.085, 0.08), + alpha_candidates_95: tuple[float, ...] = (0.05,), + min_group_sizes: tuple[int, ...] = (200, 500, 1000, 2000), + min_group_coverage_target: float = 0.88, + group_coverage_floor_target_90: float = 0.92, + max_width_budget_90: float | None = 0.80, + coverage_guardband_90: float = 0.015, + min_group_guardband_90: float = 0.0, + tuning_holdout_ratio: float = 0.20, + tuning_random_state: int = 42, + temporal_segment_floor_enabled: bool = True, + temporal_segment_freq: str = "Q", + temporal_segment_min_size: int = 250, + global_rebalance_enabled: bool = False, + global_rebalance_min_factor: float = 0.75, + global_rebalance_max_factor: float = 1.05, + global_rebalance_step: float = 0.01, + partition: str = "grade", + partition_candidates: tuple[str, ...] | None = None, + group_coverage_floor_enabled: bool = True, + shrinkback_enabled: bool = False, + group_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), + temporal_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), + artifact_namespace: str | None = None, + scaled_scores_options: tuple[bool, ...] = (True, False), + score_scale_families: tuple[str, ...] = ("bernoulli_sqrt", "none"), + partition_probability_sources: tuple[str, ...] = ("raw",), + n_score_bins_candidates: tuple[int, ...] = (10,), + fallback_modes: tuple[str, ...] = ("grade_then_global",), + calibration_fraction: float = 1.0, + evaluation_scope: str = "test", + mode: str = "search", + replay_manifest_path: str | None = None, + calibrator_override_path: str | None = None, + run_tag: str | None = None, +): + logger.info("Starting Mondrian conformal interval generation with 90% auto-tuning") + run_mode = str(mode or "search").strip().lower() or "search" + replay_cfg = manifest_section(load_replay_manifest(replay_manifest_path), "conformal") + if run_mode == "replay": + source_namespace = str(replay_cfg.get("source_namespace", "")).strip() + replay_mode = str(replay_cfg.get("replay_mode", "")).strip().lower() + if replay_mode != "restore_blessed_namespace" or not source_namespace: + raise ValueError( + "Conformal replay requires source_namespace + restore_blessed_namespace." + ) + _restore_replay_namespace(source_namespace, run_tag=run_tag) + return + + inputs = _load_conformal_inputs( + calibration_fraction=calibration_fraction, + calibrator_override_path=calibrator_override_path, + ) + model_path = inputs.model_path + cal_df = inputs.cal_df + test_df = inputs.test_df + X_cal = inputs.X_cal + y_cal = inputs.y_cal + y_test = inputs.y_test + group_cal_base = inputs.group_cal_base + group_test_base = inputs.group_test_base + y_prob_cal_raw = inputs.y_prob_cal_raw + tuning_split = _build_tuning_split( + cal_df=cal_df, + test_df=test_df, + X_cal=X_cal, + y_cal=y_cal, + group_cal_base=group_cal_base, + y_prob_cal_raw=y_prob_cal_raw, + tuning_holdout_ratio=tuning_holdout_ratio, + tuning_random_state=tuning_random_state, + ) + idx_cal_tune = tuning_split.idx_cal_tune + X_cal_fit = tuning_split.X_cal_fit + y_cal_fit = tuning_split.y_cal_fit + X_tune = tuning_split.X_tune + y_tune = tuning_split.y_tune + group_cal_fit_base = tuning_split.group_cal_fit_base + group_tune_base = tuning_split.group_tune_base + issue_tune = tuning_split.issue_tune + issue_test = tuning_split.issue_test + _log_tuning_split(inputs, tuning_split) + + target_coverage_90 = 1.0 - alpha_target_90 + group_coverage_floor_target_90 = max( + float(min_group_coverage_target), + float(group_coverage_floor_target_90), + ) + evaluation_scope_key = str(evaluation_scope or "test").strip().lower() or "test" + if evaluation_scope_key not in {"test", "holdout"}: + raise ValueError(f"Unsupported evaluation_scope: {evaluation_scope}") + tuning_grid = _resolve_tuning_grid( + partition=partition, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + scaled_scores_options=scaled_scores_options, + ) + partition_candidates = tuning_grid.partition_candidates + prob_fit_lookup, prob_tune_lookup, prob_test_lookup = _build_probability_lookups( + inputs, + tuning_split, + ) + interval_fit_pred = prob_fit_lookup["calibrated"] + interval_tune_pred = prob_tune_lookup["calibrated"] + interval_test_pred = prob_test_lookup["calibrated"] + tuning_search = _run_tuning_search( + tuning_grid=tuning_grid, + alpha_candidates_90=alpha_candidates_90, + min_group_sizes=min_group_sizes, + alpha_target_90=alpha_target_90, + target_coverage_90=target_coverage_90, + artifact_namespace=artifact_namespace, + prob_fit_lookup=prob_fit_lookup, + prob_tune_lookup=prob_tune_lookup, + prob_test_lookup=prob_test_lookup, + interval_fit_pred=interval_fit_pred, + interval_tune_pred=interval_tune_pred, + y_cal_fit=y_cal_fit, + y_tune=y_tune, + group_cal_fit_base=group_cal_fit_base, + group_tune_base=group_tune_base, + group_test_base=group_test_base, + issue_tune=issue_tune, + ) + tuning_rows = tuning_search.tuning_rows + partition_cache = tuning_search.partition_cache + + tuning_selection = _select_best_tuning_config( + tuning_rows, + partition_candidates=partition_candidates, + alpha_target_90=alpha_target_90, + min_group_coverage_target=min_group_coverage_target, + group_coverage_floor_target_90=group_coverage_floor_target_90, + coverage_guardband_90=coverage_guardband_90, + min_group_guardband_90=min_group_guardband_90, + max_width_budget_90=max_width_budget_90, + target_coverage_90=target_coverage_90, + ) + tuning_df = tuning_selection.tuning_df + best_row = tuning_selection.best_row + selection_tier = tuning_selection.selection_tier + best_cfg = tuning_selection.best_cfg + logger.info( + "Best 90% tuning config: " + f"partition={best_cfg['partition']}, " + f"prob_source={best_cfg['partition_probability_source']}, " + f"n_bins={best_cfg['n_score_bins']}, " + f"alpha_used={best_cfg['alpha_used_90']}, scaled_scores={best_cfg['scaled_scores']}, " + f"score_scale_family={best_cfg['score_scale_family']}, " + f"min_group_size={best_cfg['min_group_size']}, " + f"coverage={best_row['empirical_coverage']:.4f}, " + f"min_group_coverage={best_row['min_group_coverage']:.4f}, " + f"width={best_row['avg_interval_width']:.4f}, " + f"tier={selection_tier}" + ) + + group_cal_fit, group_test, partition_meta = _cached_partition_labels( + cache=partition_cache, + eval_scope="test", + partition_candidate=best_cfg["partition"], + partition_probability_source=best_cfg["partition_probability_source"], + n_score_bins=best_cfg["n_score_bins"], + fallback_mode=best_cfg["fallback_mode"], + min_group_size=best_cfg["min_group_size"], + prob_fit_lookup=prob_fit_lookup, + prob_tune_lookup=prob_tune_lookup, + prob_test_lookup=prob_test_lookup, + group_cal_fit_base=group_cal_fit_base, + group_tune_base=group_tune_base, + group_test_base=group_test_base, + ) + group_cal_fit_holdout, group_tune, _ = _cached_partition_labels( + cache=partition_cache, + eval_scope="tune", + partition_candidate=best_cfg["partition"], + partition_probability_source=best_cfg["partition_probability_source"], + n_score_bins=best_cfg["n_score_bins"], + fallback_mode=best_cfg["fallback_mode"], + min_group_size=best_cfg["min_group_size"], + prob_fit_lookup=prob_fit_lookup, + prob_tune_lookup=prob_tune_lookup, + prob_test_lookup=prob_test_lookup, + group_cal_fit_base=group_cal_fit_base, + group_tune_base=group_tune_base, + group_test_base=group_test_base, + ) + + evidence_90 = _build_90_interval_evidence( + evaluation_scope_key=evaluation_scope_key, + y_test=y_test, + y_tune=y_tune, + issue_test=issue_test, + issue_tune=issue_tune, + interval_fit_pred=interval_fit_pred, + interval_test_pred=interval_test_pred, + interval_tune_pred=interval_tune_pred, + y_cal_fit=y_cal_fit, + group_cal_fit=group_cal_fit, + group_test=group_test, + group_cal_fit_holdout=group_cal_fit_holdout, + group_tune=group_tune, + best_cfg=best_cfg, + alpha_target_90=alpha_target_90, + target_coverage_90=target_coverage_90, + group_coverage_floor_enabled=group_coverage_floor_enabled, + group_coverage_floor_target_90=group_coverage_floor_target_90, + group_multiplier_grid=group_multiplier_grid, + temporal_segment_floor_enabled=temporal_segment_floor_enabled, + temporal_segment_freq=temporal_segment_freq, + temporal_segment_min_size=temporal_segment_min_size, + temporal_multiplier_grid=temporal_multiplier_grid, + shrinkback_enabled=shrinkback_enabled, + min_group_coverage_target=min_group_coverage_target, + global_rebalance_enabled=global_rebalance_enabled, + global_rebalance_min_factor=global_rebalance_min_factor, + global_rebalance_max_factor=global_rebalance_max_factor, + global_rebalance_step=global_rebalance_step, + ) + y_eval_90 = evidence_90.y_eval + eval_groups_90 = evidence_90.eval_groups + y_pred_90 = evidence_90.y_pred + y_int_90 = evidence_90.y_intervals + diag_90 = evidence_90.diag + metrics_90 = evidence_90.metrics + group_metrics_90 = evidence_90.group_metrics + + best_alpha_95 = _select_alpha_95( + alpha_95=alpha_95, + alpha_candidates_95=alpha_candidates_95, + interval_fit_pred=interval_fit_pred, + interval_tune_pred=interval_tune_pred, + y_cal_fit=y_cal_fit, + y_tune=y_tune, + group_cal_fit_holdout=group_cal_fit_holdout, + group_tune=group_tune, + best_cfg=best_cfg, + ) + + evidence_95 = _build_95_interval_evidence( + evaluation_scope_key=evaluation_scope_key, + interval_fit_pred=interval_fit_pred, + interval_test_pred=interval_test_pred, + interval_tune_pred=interval_tune_pred, + y_cal_fit=y_cal_fit, + group_cal_fit=group_cal_fit, + group_test=group_test, + group_tune=group_tune, + evidence_90=evidence_90, + best_cfg=best_cfg, + best_alpha_95=best_alpha_95, + ) + y_int_95 = evidence_95.y_intervals + diag_95 = evidence_95.diag + metrics_95 = evidence_95.metrics + group_metrics_95 = evidence_95.group_metrics + + paths = _resolve_artifact_paths(artifact_namespace) + tables = _build_conformal_artifact_tables( + y_eval_90=y_eval_90, + y_pred_90=y_pred_90, + y_int_90=y_int_90, + y_int_95=y_int_95, + eval_groups_90=eval_groups_90, + eval_temporal_segments=evidence_90.eval_temporal_segments, + evaluation_scope_key=evaluation_scope_key, + test_df=test_df, + cal_df=cal_df, + idx_cal_tune=idx_cal_tune, + group_metrics_90=group_metrics_90, + group_metrics_95=group_metrics_95, + coverage_floor_report=evidence_90.coverage_floor_report, + width_attr_rows=evidence_90.width_attr_rows, + ) + payload = _build_conformal_results_payload( + model_path=model_path, + calibrator_override_path=calibrator_override_path, + metrics_90=metrics_90, + metrics_95=metrics_95, + diag_90=diag_90, + diag_95=diag_95, + partition_meta=partition_meta, + partition=partition, + group_metrics_90=group_metrics_90, + group_metrics_95=group_metrics_95, + best_cfg=best_cfg, + alpha_candidates_95=alpha_candidates_95, + best_alpha_95=best_alpha_95, + paths=paths, + group_multipliers=evidence_90.group_multipliers, + temporal_segment_floor_enabled=temporal_segment_floor_enabled, + temporal_segment_freq=temporal_segment_freq, + temporal_segment_min_size=temporal_segment_min_size, + temporal_segment_multipliers=evidence_90.temporal_segment_multipliers, + group_coverage_floor_enabled=group_coverage_floor_enabled, + shrinkback_enabled=shrinkback_enabled, + global_rebalance_diagnostics=evidence_90.global_rebalance_diagnostics, + group_coverage_floor_target_90=group_coverage_floor_target_90, + X_cal_fit=X_cal_fit, + X_tune=X_tune, + tuning_holdout_ratio=tuning_holdout_ratio, + tuning_random_state=tuning_random_state, + calibration_fraction=calibration_fraction, + evaluation_scope_key=evaluation_scope_key, + tune_metrics_90_before=evidence_90.tune_metrics_before_floor, + tune_metrics_90_after=evidence_90.tune_metrics_after_floor, + tune_metrics_90_after_temporal=evidence_90.tune_metrics_after_temporal_floor, + ) + _persist_conformal_artifacts( + paths=paths, + artifact_namespace=artifact_namespace, + run_tag=run_tag, + tables=tables, + tuning_df=tuning_df, + coverage_floor_report=evidence_90.coverage_floor_report, + temporal_segment_report=evidence_90.temporal_segment_report, + shrinkback_report=evidence_90.shrinkback_report, + payload=payload, + partition_meta=partition_meta, + partition=partition, + best_cfg=best_cfg, + best_alpha_95=best_alpha_95, + group_multipliers=evidence_90.group_multipliers, + temporal_segment_multipliers=evidence_90.temporal_segment_multipliers, + evaluation_scope_key=evaluation_scope_key, + metrics_90=metrics_90, + metrics_95=metrics_95, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( diff --git a/scripts/generate_crpto_figures.py b/scripts/generate_crpto_figures.py index debff3e..ca9b02e 100644 --- a/scripts/generate_crpto_figures.py +++ b/scripts/generate_crpto_figures.py @@ -19,6 +19,7 @@ import argparse import shutil from pathlib import Path +from typing import Any import matplotlib import matplotlib.pyplot as plt @@ -602,6 +603,62 @@ def _crpto_fig7_uncertainty_baselines() -> None: _save(fig, "crpto_fig7_uncertainty_baselines") +def _first_matching_column(df: pd.DataFrame, tokens: tuple[str, ...]) -> str | None: + return next((col for col in df.columns if any(token in col.lower() for token in tokens)), None) + + +def _alpha_pareto_column_map(df: pd.DataFrame) -> dict[str, str | None]: + return { + "variant": _first_matching_column(df, ("variant", "method")), + "alpha": _first_matching_column(df, ("alpha",)), + "coverage": _first_matching_column(df, ("coverage",)), + "width": _first_matching_column(df, ("width",)), + "eligible": _first_matching_column(df, ("eligible", "n_eligible")), + } + + +def _alpha_pareto_missing_columns(columns: dict[str, str | None]) -> list[str]: + return [key for key in ["variant", "alpha", "coverage", "width"] if not columns.get(key)] + + +def _alpha_pareto_variants(df: pd.DataFrame, variant_col: str) -> list[Any]: + return list(df[variant_col].drop_duplicates()) + + +def _alpha_pareto_variant_styles( + variants: list[Any], +) -> tuple[dict[Any, str], dict[Any, str]]: + colors = { + variant: PALETTE["blue"] if "mond" in str(variant).lower() else PALETTE["orange"] + for variant in variants + } + labels = { + variant: "Mondrian CP" if "mond" in str(variant).lower() else "Global Split-CP" + for variant in variants + } + return colors, labels + + +def _alpha_pareto_subframe( + df: pd.DataFrame, + *, + variant_col: str, + alpha_col: str, + variant: Any, +) -> pd.DataFrame: + return df[df[variant_col] == variant].sort_values(alpha_col) + + +def _alpha_annotation_offset(index: int, n_rows: int) -> tuple[int, int]: + offset_y = 4 if index % 2 == 0 else -8 + offset_x = 4 if index < n_rows - 1 else -24 + return offset_x, offset_y + + +def _alpha_tick_labels(values: pd.Series) -> list[str]: + return [str(round(float(value), 2)) for value in values] + + def _crpto_fig8_alpha_pareto() -> None: """Fig 8 — Alpha sweep Pareto: Mondrian vs Global (coverage × width × eligible loans).""" df = pd.read_parquet(DATA_DIR / "alpha_sweep_pareto_both.parquet") @@ -610,36 +667,27 @@ def _crpto_fig8_alpha_pareto() -> None: logger.warning("alpha_sweep_pareto_both empty — skipping fig8") return - # Detect variant column - variant_col = next( - (c for c in df.columns if "variant" in c.lower() or "method" in c.lower()), None - ) - alpha_col = next((c for c in df.columns if "alpha" in c.lower()), None) - cov_col = next((c for c in df.columns if "coverage" in c.lower()), None) - width_col = next((c for c in df.columns if "width" in c.lower()), None) - elig_col = next( - (c for c in df.columns if "eligible" in c.lower() or "n_eligible" in c.lower()), None - ) - - if not all([variant_col, alpha_col, cov_col, width_col]): + columns = _alpha_pareto_column_map(df) + missing = _alpha_pareto_missing_columns(columns) + if missing: logger.warning(f"alpha_sweep_pareto_both missing columns. Got: {list(df.columns)}") return + variant_col = str(columns["variant"]) + alpha_col = str(columns["alpha"]) + cov_col = str(columns["coverage"]) + width_col = str(columns["width"]) + elig_col = columns["eligible"] fig, axes = plt.subplots(1, 2, figsize=(COL2, HEIGHT_M)) - variants = df[variant_col].unique() if variant_col else ["global", "mondrian"] - var_colors = { - v: PALETTE["blue"] if "mond" in str(v).lower() else PALETTE["orange"] for v in variants - } - var_labels = { - v: "Mondrian CP" if "mond" in str(v).lower() else "Global Split-CP" for v in variants - } + variants = _alpha_pareto_variants(df, variant_col) + var_colors, var_labels = _alpha_pareto_variant_styles(variants) # Left: width vs coverage scatter (Pareto) ax = axes[0] ax.axvline(0.90, color=PALETTE["red"], lw=0.8, ls="--", alpha=0.6, label="90% target") for var in variants: - sub = df[df[variant_col] == var].sort_values(alpha_col) + sub = _alpha_pareto_subframe(df, variant_col=variant_col, alpha_col=alpha_col, variant=var) ax.plot( sub[cov_col], sub[width_col], @@ -649,8 +697,7 @@ def _crpto_fig8_alpha_pareto() -> None: label=var_labels[var], ) for idx, (_, row) in enumerate(sub.iterrows()): - offset_y = 4 if idx % 2 == 0 else -8 - offset_x = 4 if idx < len(sub) - 1 else -24 + offset_x, offset_y = _alpha_annotation_offset(idx, len(sub)) ax.annotate( f"α={row[alpha_col]:.2f}", (row[cov_col], row[width_col]), @@ -669,9 +716,14 @@ def _crpto_fig8_alpha_pareto() -> None: ax2 = axes[1] if elig_col: for var in variants: - sub = df[df[variant_col] == var].sort_values(alpha_col) + sub = _alpha_pareto_subframe( + df, + variant_col=variant_col, + alpha_col=alpha_col, + variant=var, + ) ax2.bar( - [str(round(a, 2)) for a in sub[alpha_col]], + _alpha_tick_labels(sub[alpha_col]), sub[elig_col], label=var_labels[var], color=var_colors[var], @@ -1824,13 +1876,13 @@ def _crpto_fig21_end_to_end_arc() -> None: def _crpto_fig25_price_of_robustness_scaling() -> None: - """Fig 25 — price of robustness scales with panel default rate (A34). + """Fig 25 — external price of robustness by panel default rate (A34). Frozen external applications (Freddie green/combined/red and Prosper) show a - positive premium that increases monotonically with the panel default rate. - The selected Lending Club champion is drawn as a contrasting favorable - reference line (it is a single selected point, not part of the default-rate - series). Data: models/crpto_multidataset_external_status.json. + positive premium ordered by panel default rate in the four observed cases. + The historical Lending Club field is intentionally excluded because its + stored nonrobust baseline was not a point-only comparator. Data: + models/crpto_multidataset_external_status.json. """ status = load_json(MODELS_DIR / "crpto_multidataset_external_status.json") seg_label = { @@ -1862,21 +1914,11 @@ def _crpto_fig25_price_of_robustness_scaling() -> None: x = np.array([p[1] for p in pts]) y = np.array([p[2] for p in pts]) - lc_price = -10.56 # selected Lending Club champion (frozen field) - fig, ax = plt.subplots(figsize=(COL2, HEIGHT_M)) ax.set_xscale("log") ax.set_xlim(0.4, 55) - ax.set_ylim(-13.5, 12.5) + ax.set_ylim(0.0, 11.5) ax.axhline(0.0, color=PALETTE["gray"], lw=1.0, ls="--", zorder=1) - ax.axhline( - lc_price, - color=PALETTE["orange"], - lw=1.4, - ls=":", - zorder=2, - label="Lending Club selected champion (−10.56%)", - ) ax.plot( x, y, @@ -1897,14 +1939,15 @@ def _crpto_fig25_price_of_robustness_scaling() -> None: color=PALETTE["green"], fontweight="bold", ) - ax.text(0.46, 1.2, "robustness costs a premium", fontsize=6.4, color="#666", style="italic") - ax.text(0.46, -2.6, "robustness adds value", fontsize=6.4, color="#666", style="italic") + ax.text( + 0.46, 0.45, "positive premium in all four cases", fontsize=6.4, color="#666", style="italic" + ) ax.set_xticks([0.5, 1, 2, 5, 10, 30]) ax.set_xticklabels(["0.5", "1", "2", "5", "10", "30"]) ax.tick_params(which="minor", bottom=False) ax.set_xlabel("Panel default rate (%, log scale)") ax.set_ylabel("Price of robustness (%)") - ax.set_title("Price of robustness scales with panel default risk") + ax.set_title("External robustness premium across frozen panels") ax.legend(loc="lower right", fontsize=6.6) fig.tight_layout() _save(fig, "crpto_fig25_price_of_robustness_scaling") diff --git a/scripts/generate_governance_status.py b/scripts/generate_governance_status.py index 22573db..db3f704 100644 --- a/scripts/generate_governance_status.py +++ b/scripts/generate_governance_status.py @@ -13,6 +13,7 @@ import argparse import json +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -40,6 +41,38 @@ SCHEMA_VERSION = "2026-03-06.1" +@dataclass(frozen=True) +class GovernanceThresholds: + psi_threshold: float + ks_pvalue_min: float + cvm_pvalue_min: float + c2st_auc_max: float + max_feature_breach_ratio: float + c2st_max_rows: int + score_psi_max: float + auc_delta_max: float + brier_increase_max: float + calibration_gap_delta_max: float + performance_max_rows: int + min_rank_overlap_top10: float + max_explanation_shap_psi: float + min_reason_code_stability: float + explanation_min_rows_per_slice: int + psi_bins: int + random_state: int + + +@dataclass(frozen=True) +class GovernanceOutputPaths: + drift_path: Path + status_path: Path + explanation_drift_path: Path + fairness_status_path: Path + fairness_frontier_path: Path + challenger_report_path: Path + model_shift_status_path: Path + + def _load_cfg(path: str) -> dict[str, Any]: with open(path, encoding="utf-8") as f: return yaml.safe_load(f) @@ -264,6 +297,131 @@ def _resolve_primary_threshold() -> float: return 0.5 +def _explanation_feature_columns(shap_raw: pd.DataFrame) -> list[str]: + return [c.replace("shap_", "") for c in shap_raw.columns if c.startswith("shap_")] + + +def _recent_comparison_periods( + shap_raw: pd.DataFrame, + periods: list[str], + *, + min_rows_per_slice: int, +) -> tuple[list[str], pd.DataFrame] | None: + comparison_periods: list[str] = [] + for period in reversed(periods): + comparison_periods.insert(0, period) + comparison_df = shap_raw.loc[ + shap_raw["issue_quarter"].astype(str).isin(comparison_periods) + ].copy() + if len(comparison_df) >= min_rows_per_slice: + return comparison_periods, comparison_df + return None + + +def _explanation_segment_pairs( + shap_raw: pd.DataFrame, + reference_df: pd.DataFrame, + comparison_df: pd.DataFrame, + *, + min_rows_per_slice: int, +) -> list[tuple[str, str, pd.DataFrame, pd.DataFrame]]: + segment_pairs: list[tuple[str, str, pd.DataFrame, pd.DataFrame]] = [ + ("overall", "all", reference_df, comparison_df) + ] + if "grade" not in shap_raw.columns: + return segment_pairs + for grade in sorted(shap_raw["grade"].dropna().astype(str).unique().tolist()): + ref_seg = reference_df.loc[reference_df["grade"].astype(str) == grade].copy() + cmp_seg = comparison_df.loc[comparison_df["grade"].astype(str) == grade].copy() + if len(ref_seg) < min_rows_per_slice or len(cmp_seg) < min_rows_per_slice: + continue + segment_pairs.append(("grade", grade, ref_seg, cmp_seg)) + return segment_pairs + + +def _rank_shap_features(segment: pd.DataFrame, feature_cols: list[str]) -> list[str]: + return sorted( + feature_cols, + key=lambda feature: segment[f"shap_{feature}"].abs().mean(), + reverse=True, + ) + + +def _shap_psi_details( + ref_seg: pd.DataFrame, + cmp_seg: pd.DataFrame, + *, + focus_features: list[str], +) -> list[dict[str, float | str]]: + rows: list[dict[str, float | str]] = [] + for feature in focus_features: + col = f"shap_{feature}" + if col not in ref_seg.columns or col not in cmp_seg.columns: + continue + psi = population_stability_index( + pd.to_numeric(ref_seg[col], errors="coerce").dropna().to_numpy(dtype=float), + pd.to_numeric(cmp_seg[col], errors="coerce").dropna().to_numpy(dtype=float), + n_bins=8, + ) + rows.append({"feature": feature, "psi": float(psi)}) + return rows + + +def _explanation_drift_row( + *, + segment_type: str, + segment: str, + ref_seg: pd.DataFrame, + cmp_seg: pd.DataFrame, + feature_cols: list[str], + periods: list[str], + comparison_periods: list[str], + comparison_period_label: str, + primary_threshold: float, + min_rank_overlap_top10: float, + max_shap_psi: float, + min_reason_code_stability: float, + min_rows_per_slice: int, + pd_col: str, +) -> dict[str, Any]: + ref_ranking = _rank_shap_features(ref_seg, feature_cols) + cmp_ranking = _rank_shap_features(cmp_seg, feature_cols) + overlap = rank_overlap_ratio(ref_ranking, cmp_ranking, top_k=10) + focus_features = list(dict.fromkeys(ref_ranking[:5] + cmp_ranking[:5]))[:5] + shap_psis = _shap_psi_details(ref_seg, cmp_seg, focus_features=focus_features) + max_feature_psi = max((float(row["psi"]) for row in shap_psis), default=0.0) + avg_feature_psi = float(np.mean([float(row["psi"]) for row in shap_psis])) if shap_psis else 0.0 + reason_match_rate, reason_details = dominant_reason_match_rate( + ref_seg, + cmp_seg, + ref_ranking[:10], + pd_col=pd_col, + threshold=primary_threshold, + min_rows_per_band=max(15, int(min_rows_per_slice / 4)), + ) + pass_rank = bool(overlap >= min_rank_overlap_top10) + pass_dist = bool(max_feature_psi <= max_shap_psi) + pass_reason = bool(reason_match_rate >= min_reason_code_stability) + return { + "segment_type": segment_type, + "segment": segment, + "reference_period": "|".join([p for p in periods if p not in comparison_periods]), + "comparison_period": comparison_period_label, + "reference_n": len(ref_seg), + "comparison_n": len(cmp_seg), + "rank_overlap_top10": float(overlap), + "avg_shap_psi_top5": float(avg_feature_psi), + "max_shap_psi_top5": float(max_feature_psi), + "reason_code_match_rate": float(reason_match_rate), + "pass_rank_overlap": pass_rank, + "pass_distribution_shift": pass_dist, + "pass_reason_code_stability": pass_reason, + "passed_all": bool(pass_rank and pass_dist and pass_reason), + "feature_psi_details": json.dumps(shap_psis, default=str), + "reason_code_details": json.dumps(reason_details, default=str), + } + + def _build_explanation_drift_report( shap_raw: pd.DataFrame, *, @@ -273,7 +431,7 @@ def _build_explanation_drift_report( min_reason_code_stability: float, min_rows_per_slice: int, ) -> pd.DataFrame: - feature_cols = [c.replace("shap_", "") for c in shap_raw.columns if c.startswith("shap_")] + feature_cols = _explanation_feature_columns(shap_raw) if shap_raw.empty or not feature_cols or "issue_quarter" not in shap_raw.columns: return pd.DataFrame() @@ -287,17 +445,14 @@ def _build_explanation_drift_report( if len(periods) < 2: return pd.DataFrame() - comparison_periods: list[str] = [] - comparison_df = pd.DataFrame() - for period in reversed(periods): - comparison_periods.insert(0, period) - comparison_df = shap_raw.loc[ - shap_raw["issue_quarter"].astype(str).isin(comparison_periods) - ].copy() - if len(comparison_df) >= min_rows_per_slice: - break - if len(comparison_df) < min_rows_per_slice: + comparison_slice = _recent_comparison_periods( + shap_raw, + periods, + min_rows_per_slice=min_rows_per_slice, + ) + if comparison_slice is None: return pd.DataFrame() + comparison_periods, comparison_df = comparison_slice reference_df = shap_raw.loc[ ~shap_raw["issue_quarter"].astype(str).isin(comparison_periods) @@ -306,131 +461,107 @@ def _build_explanation_drift_report( return pd.DataFrame() comparison_period_label = "|".join(comparison_periods) - segment_pairs: list[tuple[str, str, pd.DataFrame, pd.DataFrame]] = [ - ("overall", "all", reference_df, comparison_df) - ] - if "grade" in shap_raw.columns: - for grade in sorted(shap_raw["grade"].dropna().astype(str).unique().tolist()): - ref_seg = reference_df.loc[reference_df["grade"].astype(str) == grade].copy() - cmp_seg = comparison_df.loc[comparison_df["grade"].astype(str) == grade].copy() - if len(ref_seg) < min_rows_per_slice or len(cmp_seg) < min_rows_per_slice: - continue - segment_pairs.append(("grade", grade, ref_seg, cmp_seg)) - + segment_pairs = _explanation_segment_pairs( + shap_raw, + reference_df, + comparison_df, + min_rows_per_slice=min_rows_per_slice, + ) + pd_col = "pd_calibrated" if "pd_calibrated" in shap_raw.columns else "score_raw" rows: list[dict[str, Any]] = [] for segment_type, segment, ref_seg, cmp_seg in segment_pairs: - ref_ranking = sorted( - feature_cols, - key=lambda feature: ref_seg[f"shap_{feature}"].abs().mean(), - reverse=True, - ) - cmp_ranking = sorted( - feature_cols, - key=lambda feature: cmp_seg[f"shap_{feature}"].abs().mean(), - reverse=True, - ) - overlap = rank_overlap_ratio(ref_ranking, cmp_ranking, top_k=10) - focus_features = list(dict.fromkeys(ref_ranking[:5] + cmp_ranking[:5]))[:5] - shap_psis = [] - for feature in focus_features: - col = f"shap_{feature}" - if col not in ref_seg.columns or col not in cmp_seg.columns: - continue - psi = population_stability_index( - pd.to_numeric(ref_seg[col], errors="coerce").dropna().to_numpy(dtype=float), - pd.to_numeric(cmp_seg[col], errors="coerce").dropna().to_numpy(dtype=float), - n_bins=8, - ) - shap_psis.append({"feature": feature, "psi": float(psi)}) - - max_feature_psi = max((row["psi"] for row in shap_psis), default=0.0) - avg_feature_psi = float(np.mean([row["psi"] for row in shap_psis])) if shap_psis else 0.0 - reason_match_rate, reason_details = dominant_reason_match_rate( - ref_seg, - cmp_seg, - ref_ranking[:10], - pd_col="pd_calibrated" if "pd_calibrated" in shap_raw.columns else "score_raw", - threshold=primary_threshold, - min_rows_per_band=max(15, int(min_rows_per_slice / 4)), - ) - pass_rank = bool(overlap >= min_rank_overlap_top10) - pass_dist = bool(max_feature_psi <= max_shap_psi) - pass_reason = bool(reason_match_rate >= min_reason_code_stability) rows.append( - { - "segment_type": segment_type, - "segment": segment, - "reference_period": "|".join([p for p in periods if p not in comparison_periods]), - "comparison_period": comparison_period_label, - "reference_n": len(ref_seg), - "comparison_n": len(cmp_seg), - "rank_overlap_top10": float(overlap), - "avg_shap_psi_top5": float(avg_feature_psi), - "max_shap_psi_top5": float(max_feature_psi), - "reason_code_match_rate": float(reason_match_rate), - "pass_rank_overlap": pass_rank, - "pass_distribution_shift": pass_dist, - "pass_reason_code_stability": pass_reason, - "passed_all": bool(pass_rank and pass_dist and pass_reason), - "feature_psi_details": json.dumps(shap_psis, default=str), - "reason_code_details": json.dumps(reason_details, default=str), - } + _explanation_drift_row( + segment_type=segment_type, + segment=segment, + ref_seg=ref_seg, + cmp_seg=cmp_seg, + feature_cols=feature_cols, + periods=periods, + comparison_periods=comparison_periods, + comparison_period_label=comparison_period_label, + primary_threshold=primary_threshold, + min_rank_overlap_top10=min_rank_overlap_top10, + max_shap_psi=max_shap_psi, + min_reason_code_stability=min_reason_code_stability, + min_rows_per_slice=min_rows_per_slice, + pd_col=pd_col, + ) ) return pd.DataFrame(rows) -def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = None) -> None: - cfg = _load_cfg(config_path) - semantics = load_threshold_semantics() - resolved_run_tag = resolve_run_tag( - run_tag, - fallback_candidates=[semantics.get("run_tag"), resolve_official_baseline_run_tag()], - require_explicit=True, +def _resolve_thresholds( + triggers: dict[str, Any], + checks: dict[str, Any], +) -> GovernanceThresholds: + return GovernanceThresholds( + psi_threshold=float(triggers.get("psi_threshold", 0.25)), + ks_pvalue_min=float(checks.get("ks_pvalue_min", 0.01)), + cvm_pvalue_min=float(checks.get("cvm_pvalue_min", 0.01)), + c2st_auc_max=float(checks.get("c2st_auc_max", 0.60)), + max_feature_breach_ratio=float(checks.get("max_feature_breach_ratio", 0.15)), + c2st_max_rows=int(checks.get("c2st_max_rows_per_split", 50_000)), + score_psi_max=float(checks.get("score_psi_max", 0.15)), + auc_delta_max=float(checks.get("auc_delta_max", 0.05)), + brier_increase_max=float(checks.get("brier_increase_max", 0.02)), + calibration_gap_delta_max=float(checks.get("calibration_gap_delta_max", 0.02)), + performance_max_rows=int(checks.get("performance_max_rows_per_split", 100_000)), + min_rank_overlap_top10=float(checks.get("explanation_rank_overlap_top10_min", 0.60)), + max_explanation_shap_psi=float(checks.get("explanation_shap_psi_max", 0.25)), + min_reason_code_stability=float(checks.get("reason_code_stability_min", 0.55)), + explanation_min_rows_per_slice=int(checks.get("explanation_min_rows_per_slice", 80)), + psi_bins=int(checks.get("psi_bins", 10)), + random_state=int(checks.get("random_state", 42)), ) - triggers = cfg.get("retraining_triggers", {}) - checks = cfg.get("governance_checks", {}) - outputs = cfg.get("governance_output", {}) - psi_threshold = float(triggers.get("psi_threshold", 0.25)) - ks_pvalue_min = float(checks.get("ks_pvalue_min", 0.01)) - cvm_pvalue_min = float(checks.get("cvm_pvalue_min", 0.01)) - c2st_auc_max = float(checks.get("c2st_auc_max", 0.60)) - max_feature_breach_ratio = float(checks.get("max_feature_breach_ratio", 0.15)) - c2st_max_rows = int(checks.get("c2st_max_rows_per_split", 50_000)) - score_psi_max = float(checks.get("score_psi_max", 0.15)) - auc_delta_max = float(checks.get("auc_delta_max", 0.05)) - brier_increase_max = float(checks.get("brier_increase_max", 0.02)) - calibration_gap_delta_max = float(checks.get("calibration_gap_delta_max", 0.02)) - performance_max_rows = int(checks.get("performance_max_rows_per_split", 100_000)) - min_rank_overlap_top10 = float(checks.get("explanation_rank_overlap_top10_min", 0.60)) - max_explanation_shap_psi = float(checks.get("explanation_shap_psi_max", 0.25)) - min_reason_code_stability = float(checks.get("reason_code_stability_min", 0.55)) - explanation_min_rows_per_slice = int(checks.get("explanation_min_rows_per_slice", 80)) - - drift_path = Path( - outputs.get("drift_monitoring_path", "data/processed/drift_monitoring.parquet") - ) - status_path = Path(outputs.get("governance_status_path", "models/governance_status.json")) - explanation_drift_path = Path( - outputs.get("explanation_drift_path", "data/processed/explanation_drift.parquet") - ) - fairness_status_path = Path( - outputs.get("fairness_status_path", "models/fairness_audit_status.json") - ) - fairness_frontier_path = Path( - outputs.get( +def _output_path(outputs: dict[str, Any], key: str, default: str) -> Path: + value = outputs.get(key, default) + return Path(str(default if value is None else value)) + + +def _resolve_output_paths(outputs: dict[str, Any]) -> GovernanceOutputPaths: + return GovernanceOutputPaths( + drift_path=_output_path( + outputs, + "drift_monitoring_path", + "data/processed/drift_monitoring.parquet", + ), + status_path=_output_path( + outputs, + "governance_status_path", + "models/governance_status.json", + ), + explanation_drift_path=_output_path( + outputs, + "explanation_drift_path", + "data/processed/explanation_drift.parquet", + ), + fairness_status_path=_output_path( + outputs, + "fairness_status_path", + "models/fairness_audit_status.json", + ), + fairness_frontier_path=_output_path( + outputs, "fairness_frontier_path", "data/processed/fairness_threshold_frontier.parquet", - ) - ) - challenger_report_path = Path( - outputs.get("challenger_promotion_report_path", "models/challenger_promotion_report.json") - ) - model_shift_status_path = Path( - outputs.get("model_shift_status_path", "models/model_shift_status.json") + ), + challenger_report_path=_output_path( + outputs, + "challenger_promotion_report_path", + "models/challenger_promotion_report.json", + ), + model_shift_status_path=_output_path( + outputs, + "model_shift_status_path", + "models/model_shift_status.json", + ), ) + +def _load_governance_frames() -> tuple[pd.DataFrame, pd.DataFrame]: train_df = read_split_with_fe_fallback("data/processed/train_fe.parquet") test_df = read_split_with_fe_fallback("data/processed/test_fe.parquet") training_record = _load_training_record() @@ -442,31 +573,44 @@ def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = Non regime_meta.get("mode", regime_cfg.get("mode", "standard")), len(train_df), ) + return train_df, test_df - features = _resolve_numeric_features(train_df, test_df) - if not features: - raise ValueError("No numeric features available for governance drift checks.") - logger.info("Governance drift checks on {} numeric features", len(features)) +def _write_parquet(df: pd.DataFrame, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(path, index=False) - drift_df = drift_monitoring_report( - train_df=train_df, - test_df=test_df, - features=features, - psi_threshold=psi_threshold, - ks_pvalue_threshold=ks_pvalue_min, - cvm_pvalue_threshold=cvm_pvalue_min, - n_bins=int(checks.get("psi_bins", 10)), - ) - c2st = classifier_two_sample_test( - train_df=train_df, - test_df=test_df, - features=features, - max_rows_per_split=c2st_max_rows, - random_state=int(checks.get("random_state", 42)), +def _load_json_dict(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + return payload if isinstance(payload, dict) else {} + + +def _build_explanation_drift_if_available(thresholds: GovernanceThresholds) -> pd.DataFrame: + shap_raw_path = Path("data/processed/shap_raw_top20.parquet") + if not shap_raw_path.exists(): + return pd.DataFrame() + return _build_explanation_drift_report( + pd.read_parquet(shap_raw_path), + primary_threshold=_resolve_primary_threshold(), + min_rank_overlap_top10=thresholds.min_rank_overlap_top10, + max_shap_psi=thresholds.max_explanation_shap_psi, + min_reason_code_stability=thresholds.min_reason_code_stability, + min_rows_per_slice=thresholds.explanation_min_rows_per_slice, ) + +def _drift_breach_metrics( + drift_df: pd.DataFrame, + c2st: dict[str, Any], + performance_report: dict[str, float], + thresholds: GovernanceThresholds, +) -> dict[str, Any]: n_features = len(drift_df) psi_breaches = ( int((~drift_df.get("pass_psi", pd.Series(dtype=bool))).sum()) if n_features else 0 @@ -477,81 +621,77 @@ def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = Non ) feature_breach_ratio = float(psi_breaches / max(n_features, 1)) distribution_warning_ratio = float((ks_breaches + cvm_breaches) / max(n_features * 2, 1)) - - max_psi = float(drift_df["psi"].max()) if n_features else 0.0 - mean_psi = _safe_mean(drift_df["psi"]) if n_features else 0.0 - min_ks_pvalue = float(drift_df["ks_pvalue"].min()) if n_features else 1.0 - min_cvm_pvalue = float(drift_df["cvm_pvalue"].min()) if n_features else 1.0 - c2st_auc = _safe_float_value(c2st["c2st_auc"]) - c2st_materiality = str(c2st.get("materiality", "none")) - c2st_effective_driver_count = _safe_int_value(c2st.get("effective_driver_count", 0)) - c2st_top_drivers = _safe_list_value(c2st.get("top_drivers", [])) - performance_report = _score_and_performance_report( - train_df, - test_df, - random_state=int(checks.get("random_state", 42)), - max_rows_per_split=performance_max_rows, - psi_bins=int(checks.get("psi_bins", 10)), - ) score_psi = float(performance_report.get("score_psi", 0.0)) auc_delta = float(performance_report.get("auc_delta_train_to_test", 0.0)) brier_increase = float(performance_report.get("brier_increase_train_to_test", 0.0)) calibration_gap_delta = float(performance_report.get("calibration_gap_delta", 0.0)) - - pass_psi = bool(max_psi <= psi_threshold) - pass_breach_ratio = bool(feature_breach_ratio <= max_feature_breach_ratio) - pass_score_psi = bool(score_psi <= score_psi_max) - pass_auc_delta = bool(auc_delta <= auc_delta_max) - pass_brier_increase = bool(brier_increase <= brier_increase_max) - pass_calibration_gap_delta = bool(calibration_gap_delta <= calibration_gap_delta_max) - pass_c2st = bool(c2st_auc <= c2st_auc_max) - model_shift = interpret_model_shift( - c2st_auc=c2st_auc, - c2st_materiality=c2st_materiality, - score_psi=score_psi, - auc_delta=auc_delta, - brier_increase=brier_increase, - calibration_gap_delta=calibration_gap_delta, - distribution_warning_ratio=distribution_warning_ratio, - score_psi_max=score_psi_max, - auc_delta_max=auc_delta_max, - brier_increase_max=brier_increase_max, - calibration_gap_delta_max=calibration_gap_delta_max, + pass_psi = bool( + (float(drift_df["psi"].max()) if n_features else 0.0) <= thresholds.psi_threshold ) + pass_breach_ratio = bool(feature_breach_ratio <= thresholds.max_feature_breach_ratio) + pass_score_psi = bool(score_psi <= thresholds.score_psi_max) + pass_auc_delta = bool(auc_delta <= thresholds.auc_delta_max) + pass_brier_increase = bool(brier_increase <= thresholds.brier_increase_max) + pass_calibration_gap_delta = bool(calibration_gap_delta <= thresholds.calibration_gap_delta_max) + return { + "n_features": n_features, + "psi_breaches": psi_breaches, + "ks_breaches": ks_breaches, + "cvm_breaches": cvm_breaches, + "feature_breach_ratio": feature_breach_ratio, + "distribution_warning_ratio": distribution_warning_ratio, + "max_psi": float(drift_df["psi"].max()) if n_features else 0.0, + "mean_psi": _safe_mean(drift_df["psi"]) if n_features else 0.0, + "min_ks_pvalue": float(drift_df["ks_pvalue"].min()) if n_features else 1.0, + "min_cvm_pvalue": float(drift_df["cvm_pvalue"].min()) if n_features else 1.0, + "c2st_auc": _safe_float_value(c2st["c2st_auc"]), + "c2st_materiality": str(c2st.get("materiality", "none")), + "c2st_effective_driver_count": _safe_int_value(c2st.get("effective_driver_count", 0)), + "c2st_top_drivers": _safe_list_value(c2st.get("top_drivers", [])), + "c2st_rows_used": _safe_int_value(c2st.get("n_rows", 0)), + "score_psi": score_psi, + "auc_delta": auc_delta, + "brier_increase": brier_increase, + "calibration_gap_delta": calibration_gap_delta, + "pass_psi": pass_psi, + "pass_breach_ratio": pass_breach_ratio, + "pass_score_psi": pass_score_psi, + "pass_auc_delta": pass_auc_delta, + "pass_brier_increase": pass_brier_increase, + "pass_calibration_gap_delta": pass_calibration_gap_delta, + "pass_predictive_drift": bool( + pass_psi + and pass_breach_ratio + and pass_score_psi + and pass_auc_delta + and pass_brier_increase + and pass_calibration_gap_delta + ), + "pass_c2st": bool(_safe_float_value(c2st["c2st_auc"]) <= thresholds.c2st_auc_max), + "performance_report": performance_report, + } - drift_path.parent.mkdir(parents=True, exist_ok=True) - drift_df.to_parquet(drift_path, index=False) - shap_raw_path = Path("data/processed/shap_raw_top20.parquet") - explanation_drift = pd.DataFrame() - if shap_raw_path.exists(): - explanation_drift = _build_explanation_drift_report( - pd.read_parquet(shap_raw_path), - primary_threshold=_resolve_primary_threshold(), - min_rank_overlap_top10=min_rank_overlap_top10, - max_shap_psi=max_explanation_shap_psi, - min_reason_code_stability=min_reason_code_stability, - min_rows_per_slice=explanation_min_rows_per_slice, - ) - explanation_drift_path.parent.mkdir(parents=True, exist_ok=True) - explanation_drift.to_parquet(explanation_drift_path, index=False) +def _interpret_governance_shift( + metrics: dict[str, Any], + thresholds: GovernanceThresholds, +) -> dict[str, Any]: + return interpret_model_shift( + c2st_auc=float(metrics["c2st_auc"]), + c2st_materiality=str(metrics["c2st_materiality"]), + score_psi=float(metrics["score_psi"]), + auc_delta=float(metrics["auc_delta"]), + brier_increase=float(metrics["brier_increase"]), + calibration_gap_delta=float(metrics["calibration_gap_delta"]), + distribution_warning_ratio=float(metrics["distribution_warning_ratio"]), + score_psi_max=thresholds.score_psi_max, + auc_delta_max=thresholds.auc_delta_max, + brier_increase_max=thresholds.brier_increase_max, + calibration_gap_delta_max=thresholds.calibration_gap_delta_max, + ) - fairness_status = {} - if fairness_status_path.exists(): - try: - fairness_status = json.loads(fairness_status_path.read_text(encoding="utf-8")) - except Exception: - fairness_status = {} - fairness_pass = bool(fairness_status.get("overall_pass", False)) - - challenger_report = {} - if challenger_report_path.exists(): - try: - challenger_report = json.loads(challenger_report_path.read_text(encoding="utf-8")) - except Exception: - challenger_report = {} - challenger_promotable = bool(challenger_report.get("challenger_promotable", False)) +def _explanation_passes(explanation_drift: pd.DataFrame) -> dict[str, bool]: explainability_pass = bool( (not explanation_drift.empty) and explanation_drift["passed_all"].astype(bool).all() ) @@ -559,129 +699,154 @@ def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = Non (not explanation_drift.empty) and explanation_drift["pass_reason_code_stability"].astype(bool).all() ) - predictive_drift_pass = bool( - pass_psi - and pass_breach_ratio - and pass_score_psi - and pass_auc_delta - and pass_brier_increase - and pass_calibration_gap_delta - ) - overall_pass = bool(predictive_drift_pass and fairness_pass) - warning_flags = { - "warn_c2st": bool(not pass_c2st), - "warn_distribution_tests": bool(ks_breaches > 0 or cvm_breaches > 0), - "warn_explainability": bool(not explainability_pass), - "warn_reason_code_stability": bool(not reason_code_stability_pass), + return { + "explainability_pass": explainability_pass, + "reason_code_stability_pass": reason_code_stability_pass, } - top_breaches = drift_df.head(10).to_dict(orient="records") if n_features else [] - top_explanation_breaches = ( + +def _series_min_or_zero(df: pd.DataFrame, column: str) -> float: + return float(df[column].min()) if not df.empty else 0.0 + + +def _series_max_or_zero(df: pd.DataFrame, column: str) -> float: + return float(df[column].max()) if not df.empty else 0.0 + + +def _top_explanation_breaches(explanation_drift: pd.DataFrame) -> list[dict[str, Any]]: + if explanation_drift.empty: + return [] + return ( explanation_drift.sort_values( ["passed_all", "max_shap_psi_top5", "rank_overlap_top10"], ascending=[True, False, True], ) .head(10) .to_dict(orient="records") - if not explanation_drift.empty - else [] ) - status = { + + +def _fairness_primary_threshold(fairness_status: dict[str, Any]) -> float: + if not fairness_status: + return 0.5 + return _safe_float_value( + fairness_status.get("primary_threshold", fairness_status.get("prediction_threshold", 0.5)) + ) + + +def _warning_flags( + metrics: dict[str, Any], + explanation_checks: dict[str, bool], +) -> dict[str, bool]: + return { + "warn_c2st": bool(not metrics["pass_c2st"]), + "warn_distribution_tests": bool(metrics["ks_breaches"] > 0 or metrics["cvm_breaches"] > 0), + "warn_explainability": bool(not explanation_checks["explainability_pass"]), + "warn_reason_code_stability": bool(not explanation_checks["reason_code_stability_pass"]), + } + + +def _build_governance_status( + *, + config_path: str, + resolved_run_tag: str, + paths: GovernanceOutputPaths, + thresholds: GovernanceThresholds, + drift_df: pd.DataFrame, + explanation_drift: pd.DataFrame, + fairness_status: dict[str, Any], + challenger_report: dict[str, Any], + metrics: dict[str, Any], + model_shift: dict[str, Any], +) -> dict[str, Any]: + fairness_pass = bool(fairness_status.get("overall_pass", False)) + challenger_promotable = bool(challenger_report.get("challenger_promotable", False)) + explanation_checks = _explanation_passes(explanation_drift) + warning_flags = _warning_flags(metrics, explanation_checks) + overall_pass = bool(metrics["pass_predictive_drift"] and fairness_pass) + top_breaches = drift_df.head(10).to_dict(orient="records") if int(metrics["n_features"]) else [] + primary_threshold = _fairness_primary_threshold(fairness_status) + + return { "overall_pass": overall_pass, "checks": { - "pass_psi": pass_psi, - "pass_breach_ratio": pass_breach_ratio, - "pass_score_psi": pass_score_psi, - "pass_auc_delta": pass_auc_delta, - "pass_brier_increase": pass_brier_increase, - "pass_calibration_gap_delta": pass_calibration_gap_delta, - "pass_predictive_drift": predictive_drift_pass, + "pass_psi": bool(metrics["pass_psi"]), + "pass_breach_ratio": bool(metrics["pass_breach_ratio"]), + "pass_score_psi": bool(metrics["pass_score_psi"]), + "pass_auc_delta": bool(metrics["pass_auc_delta"]), + "pass_brier_increase": bool(metrics["pass_brier_increase"]), + "pass_calibration_gap_delta": bool(metrics["pass_calibration_gap_delta"]), + "pass_predictive_drift": bool(metrics["pass_predictive_drift"]), "pass_fairness": fairness_pass, - "pass_c2st": pass_c2st, - "pass_explainability": explainability_pass, - "pass_reason_code_stability": reason_code_stability_pass, + "pass_c2st": bool(metrics["pass_c2st"]), + "pass_explainability": explanation_checks["explainability_pass"], + "pass_reason_code_stability": explanation_checks["reason_code_stability_pass"], **warning_flags, }, "thresholds": { - "psi_threshold": psi_threshold, - "ks_pvalue_min": ks_pvalue_min, - "cvm_pvalue_min": cvm_pvalue_min, - "c2st_auc_max": c2st_auc_max, - "max_feature_breach_ratio": max_feature_breach_ratio, - "score_psi_max": score_psi_max, - "auc_delta_max": auc_delta_max, - "brier_increase_max": brier_increase_max, - "calibration_gap_delta_max": calibration_gap_delta_max, - "explanation_rank_overlap_top10_min": min_rank_overlap_top10, - "explanation_shap_psi_max": max_explanation_shap_psi, - "reason_code_stability_min": min_reason_code_stability, + "psi_threshold": thresholds.psi_threshold, + "ks_pvalue_min": thresholds.ks_pvalue_min, + "cvm_pvalue_min": thresholds.cvm_pvalue_min, + "c2st_auc_max": thresholds.c2st_auc_max, + "max_feature_breach_ratio": thresholds.max_feature_breach_ratio, + "score_psi_max": thresholds.score_psi_max, + "auc_delta_max": thresholds.auc_delta_max, + "brier_increase_max": thresholds.brier_increase_max, + "calibration_gap_delta_max": thresholds.calibration_gap_delta_max, + "explanation_rank_overlap_top10_min": thresholds.min_rank_overlap_top10, + "explanation_shap_psi_max": thresholds.max_explanation_shap_psi, + "reason_code_stability_min": thresholds.min_reason_code_stability, }, "summary": { - "n_features": n_features, - "max_psi": max_psi, - "mean_psi": mean_psi, - "min_ks_pvalue": min_ks_pvalue, - "min_cvm_pvalue": min_cvm_pvalue, - "c2st_auc": c2st_auc, - "psi_breaches": psi_breaches, - "ks_breaches": ks_breaches, - "cvm_breaches": cvm_breaches, - "feature_breach_ratio": feature_breach_ratio, - "distribution_warning_ratio": distribution_warning_ratio, - "c2st_rows_used": _safe_int_value(c2st.get("n_rows", 0)), - "c2st_materiality": c2st_materiality, - "c2st_effective_driver_count": c2st_effective_driver_count, - **performance_report, + "n_features": int(metrics["n_features"]), + "max_psi": float(metrics["max_psi"]), + "mean_psi": float(metrics["mean_psi"]), + "min_ks_pvalue": float(metrics["min_ks_pvalue"]), + "min_cvm_pvalue": float(metrics["min_cvm_pvalue"]), + "c2st_auc": float(metrics["c2st_auc"]), + "psi_breaches": int(metrics["psi_breaches"]), + "ks_breaches": int(metrics["ks_breaches"]), + "cvm_breaches": int(metrics["cvm_breaches"]), + "feature_breach_ratio": float(metrics["feature_breach_ratio"]), + "distribution_warning_ratio": float(metrics["distribution_warning_ratio"]), + "c2st_rows_used": int(metrics["c2st_rows_used"]), + "c2st_materiality": str(metrics["c2st_materiality"]), + "c2st_effective_driver_count": int(metrics["c2st_effective_driver_count"]), + **metrics["performance_report"], "n_explanation_segments": len(explanation_drift), - "min_rank_overlap_top10": float(explanation_drift["rank_overlap_top10"].min()) - if not explanation_drift.empty - else 0.0, - "max_explanation_shap_psi": float(explanation_drift["max_shap_psi_top5"].max()) - if not explanation_drift.empty - else 0.0, - "min_reason_code_stability": float(explanation_drift["reason_code_match_rate"].min()) - if not explanation_drift.empty - else 0.0, + "min_rank_overlap_top10": _series_min_or_zero(explanation_drift, "rank_overlap_top10"), + "max_explanation_shap_psi": _series_max_or_zero(explanation_drift, "max_shap_psi_top5"), + "min_reason_code_stability": _series_min_or_zero( + explanation_drift, "reason_code_match_rate" + ), "fairness_overall_pass": fairness_pass, - "fairness_primary_threshold": _safe_float_value( - fairness_status.get( - "primary_threshold", fairness_status.get("prediction_threshold", 0.5) - ) - ) - if fairness_status - else 0.5, + "fairness_primary_threshold": primary_threshold, "challenger_promotable": challenger_promotable, "model_shift_type": str(model_shift["shift_type"]), "governance_posture": str(model_shift["governance_posture"]), }, "warnings": warning_flags, "c2st": { - "auc": c2st_auc, - "materiality": c2st_materiality, - "effective_driver_count": c2st_effective_driver_count, - "top_drivers": c2st_top_drivers, + "auc": float(metrics["c2st_auc"]), + "materiality": str(metrics["c2st_materiality"]), + "effective_driver_count": int(metrics["c2st_effective_driver_count"]), + "top_drivers": metrics["c2st_top_drivers"], }, "model_shift": model_shift, "artifacts": { - "drift_monitoring_path": str(drift_path), - "explanation_drift_path": str(explanation_drift_path), - "fairness_status_path": str(fairness_status_path), - "fairness_frontier_path": str(fairness_frontier_path), - "challenger_promotion_report_path": str(challenger_report_path), - "model_shift_status_path": str(model_shift_status_path), + "drift_monitoring_path": str(paths.drift_path), + "explanation_drift_path": str(paths.explanation_drift_path), + "fairness_status_path": str(paths.fairness_status_path), + "fairness_frontier_path": str(paths.fairness_frontier_path), + "challenger_promotion_report_path": str(paths.challenger_report_path), + "model_shift_status_path": str(paths.model_shift_status_path), }, "top_drift_features": top_breaches, - "top_explanation_breaches": top_explanation_breaches, - "primary_threshold": _safe_float_value( - fairness_status.get( - "primary_threshold", fairness_status.get("prediction_threshold", 0.5) - ) - ) - if fairness_status - else 0.5, - "explainability_pass": explainability_pass, - "explanation_drift_pass": explainability_pass, - "reason_code_stability_pass": reason_code_stability_pass, + "top_explanation_breaches": _top_explanation_breaches(explanation_drift), + "primary_threshold": primary_threshold, + "explainability_pass": explanation_checks["explainability_pass"], + "explanation_drift_pass": explanation_checks["explainability_pass"], + "reason_code_stability_pass": explanation_checks["reason_code_stability_pass"], "challenger_promotable": challenger_promotable, "policy_config": config_path, **build_artifact_metadata( @@ -691,41 +856,131 @@ def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = Non ), } - status_path.parent.mkdir(parents=True, exist_ok=True) - with open(status_path, "w", encoding="utf-8") as f: - json.dump(status, f, indent=2) - - model_shift_status_path.parent.mkdir(parents=True, exist_ok=True) - model_shift_status_path.write_text( - json.dumps( - { - "diagnostic_only": True, - "overall_pass": bool(model_shift["governance_posture"] != "candidate_gate"), - "summary": model_shift, - "artifacts": {"governance_status_path": str(status_path)}, - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - }, - indent=2, - ), - encoding="utf-8", + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + +def _write_model_shift_status( + *, + path: Path, + status_path: Path, + model_shift: dict[str, Any], + resolved_run_tag: str, +) -> None: + _write_json( + path, + { + "diagnostic_only": True, + "overall_pass": bool(model_shift["governance_posture"] != "candidate_gate"), + "summary": model_shift, + "artifacts": {"governance_status_path": str(status_path)}, + **build_artifact_metadata( + schema_version=SCHEMA_VERSION, + run_tag=resolved_run_tag, + require_explicit=True, + ), + }, ) - logger.info("Saved drift monitoring: {}", drift_path) - logger.info("Saved governance status: {}", status_path) - logger.info("Saved model-shift interpretation: {}", model_shift_status_path) + +def _log_governance_outputs( + *, + paths: GovernanceOutputPaths, + status: dict[str, Any], + metrics: dict[str, Any], +) -> None: + logger.info("Saved drift monitoring: {}", paths.drift_path) + logger.info("Saved governance status: {}", paths.status_path) + logger.info("Saved model-shift interpretation: {}", paths.model_shift_status_path) logger.info( "Governance checks pass={} (max_psi={:.4f}, score_psi={:.4f}, auc_delta={:.4f}, brier_increase={:.4f}, c2st_auc={:.4f})", - overall_pass, - max_psi, - score_psi, - auc_delta, - brier_increase, - c2st_auc, + status["overall_pass"], + metrics["max_psi"], + metrics["score_psi"], + metrics["auc_delta"], + metrics["brier_increase"], + metrics["c2st_auc"], + ) + + +def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = None) -> None: + cfg = _load_cfg(config_path) + semantics = load_threshold_semantics() + resolved_run_tag = resolve_run_tag( + run_tag, + fallback_candidates=[semantics.get("run_tag"), resolve_official_baseline_run_tag()], + require_explicit=True, + ) + + triggers = cfg.get("retraining_triggers", {}) + checks = cfg.get("governance_checks", {}) + outputs = cfg.get("governance_output", {}) + thresholds = _resolve_thresholds(triggers, checks) + paths = _resolve_output_paths(outputs) + + train_df, test_df = _load_governance_frames() + + features = _resolve_numeric_features(train_df, test_df) + if not features: + raise ValueError("No numeric features available for governance drift checks.") + + logger.info("Governance drift checks on {} numeric features", len(features)) + + drift_df = drift_monitoring_report( + train_df=train_df, + test_df=test_df, + features=features, + psi_threshold=thresholds.psi_threshold, + ks_pvalue_threshold=thresholds.ks_pvalue_min, + cvm_pvalue_threshold=thresholds.cvm_pvalue_min, + n_bins=thresholds.psi_bins, + ) + + c2st = classifier_two_sample_test( + train_df=train_df, + test_df=test_df, + features=features, + max_rows_per_split=thresholds.c2st_max_rows, + random_state=thresholds.random_state, + ) + performance_report = _score_and_performance_report( + train_df, + test_df, + random_state=thresholds.random_state, + max_rows_per_split=thresholds.performance_max_rows, + psi_bins=thresholds.psi_bins, + ) + metrics = _drift_breach_metrics(drift_df, c2st, performance_report, thresholds) + model_shift = _interpret_governance_shift(metrics, thresholds) + + _write_parquet(drift_df, paths.drift_path) + explanation_drift = _build_explanation_drift_if_available(thresholds) + _write_parquet(explanation_drift, paths.explanation_drift_path) + + fairness_status = _load_json_dict(paths.fairness_status_path) + challenger_report = _load_json_dict(paths.challenger_report_path) + status = _build_governance_status( + config_path=config_path, + resolved_run_tag=resolved_run_tag, + paths=paths, + thresholds=thresholds, + drift_df=drift_df, + explanation_drift=explanation_drift, + fairness_status=fairness_status, + challenger_report=challenger_report, + metrics=metrics, + model_shift=model_shift, + ) + _write_json(paths.status_path, status) + _write_model_shift_status( + path=paths.model_shift_status_path, + status_path=paths.status_path, + model_shift=model_shift, + resolved_run_tag=resolved_run_tag, ) + _log_governance_outputs(paths=paths, status=status, metrics=metrics) if __name__ == "__main__": diff --git a/scripts/generate_mrm_report.py b/scripts/generate_mrm_report.py index beb74f8..92627a1 100644 --- a/scripts/generate_mrm_report.py +++ b/scripts/generate_mrm_report.py @@ -145,20 +145,18 @@ def _build_skops_sidecar( card.add_metrics( section="Validation", description="Pipeline-level subsystem gates consumed by MRM.", - metrics={ - "pipeline_overall_pass": bool( - statuses.get("pipeline", {}).get("overall_pass", False) - ), - "conformal_overall_pass": bool( - statuses.get("conformal", {}).get("overall_pass", False) - ), - "governance_overall_pass": bool( - statuses.get("governance", {}).get("overall_pass", False) - ), - "fairness_overall_pass": bool( - statuses.get("fairness", {}).get("overall_pass", False) - ), - }, + pipeline_overall_pass=int( + bool(statuses.get("pipeline", {}).get("overall_pass", False)) + ), + conformal_overall_pass=int( + bool(statuses.get("conformal", {}).get("overall_pass", False)) + ), + governance_overall_pass=int( + bool(statuses.get("governance", {}).get("overall_pass", False)) + ), + fairness_overall_pass=int( + bool(statuses.get("fairness", {}).get("overall_pass", False)) + ), ) card.add_hyperparams() model_card_md = card.render() diff --git a/scripts/optimize_portfolio.py b/scripts/optimize_portfolio.py index 418d306..315feeb 100644 --- a/scripts/optimize_portfolio.py +++ b/scripts/optimize_portfolio.py @@ -15,7 +15,11 @@ from loguru import logger from src.models.conformal_artifacts import load_conformal_intervals -from src.optimization.portfolio_model import optimize_portfolio_allocation +from src.optimization.input_alignment import align_candidate_intervals +from src.optimization.portfolio_model import ( + optimize_portfolio_allocation, + solution_allocation_vector, +) from src.optimization.robust_opt import scenario_analysis from src.utils.pipeline_runtime import ( atomic_write_parquet, @@ -76,55 +80,33 @@ def _align_candidates_and_intervals( random_state: int = 42, ) -> tuple[pd.DataFrame, np.ndarray, np.ndarray, np.ndarray]: """Align loans with PD intervals using ID when available, with legacy fallback.""" - col_point, col_low, col_high = _resolve_interval_columns(intervals) - max_candidates_norm = ( - None if max_candidates is None or int(max_candidates) <= 0 else int(max_candidates) + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=max_candidates, + random_state=random_state, ) - base_n = min(len(candidates), len(intervals)) - if max_candidates_norm is not None: - base_n = min(base_n, max_candidates_norm) - - if "id" in candidates.columns and "id" in intervals.columns: - cand = candidates.copy() - ints = intervals.copy() - cand["_id_join"] = cand["id"].astype(str) - ints["_id_join"] = ints["id"].astype(str) - ints = ints.drop_duplicates(subset="_id_join", keep="first") - merged = cand.merge( - ints[["_id_join", col_point, col_low, col_high]], - on="_id_join", - how="inner", - ) - if merged.empty: - raise ValueError("ID-based merge between candidates and intervals returned zero rows.") - n = len(merged) if max_candidates_norm is None else min(len(merged), max_candidates_norm) - if len(merged) > n: - rng = np.random.default_rng(random_state) - idx = np.sort(rng.choice(np.arange(len(merged)), size=n, replace=False)) - merged = merged.iloc[idx].reset_index(drop=True) - else: - merged = merged.reset_index(drop=True) - - aligned_loans = merged[candidates.columns].copy() - pd_point = merged[col_point].to_numpy(dtype=float) - pd_low = merged[col_low].to_numpy(dtype=float) - pd_high = merged[col_high].to_numpy(dtype=float) - logger.info( - f"Aligned candidates and intervals by id: n={len(aligned_loans):,} " - f"(candidate_rows={len(candidates):,}, interval_rows={len(intervals):,})" + col_point, col_low, col_high = _resolve_interval_columns(aligned.intervals) + if aligned.mode == "position": + logger.warning( + "Conformal interval artifact has no id or _row_number alignment key; " + "using reproducible positional sampling for optimization." ) - return aligned_loans, pd_point, pd_low, pd_high - - # Legacy fallback where interval artifact has no stable key. - logger.warning( - "Conformal interval artifact has no id alignment key; using positional fallback for optimization." + logger.info( + "Aligned candidates and intervals by {}: n={:,} " + "(alignable_rows={:,}, candidate_rows={:,}, interval_rows={:,})", + aligned.mode, + aligned.selected_rows, + aligned.available_rows, + len(candidates), + len(intervals), + ) + return ( + aligned.candidates, + aligned.intervals[col_point].to_numpy(dtype=float), + aligned.intervals[col_low].to_numpy(dtype=float), + aligned.intervals[col_high].to_numpy(dtype=float), ) - aligned_loans = candidates.head(base_n).reset_index(drop=True).copy() - ints = intervals.head(base_n).reset_index(drop=True) - pd_point = ints[col_point].to_numpy(dtype=float) - pd_low = ints[col_low].to_numpy(dtype=float) - pd_high = ints[col_high].to_numpy(dtype=float) - return aligned_loans, pd_point, pd_low, pd_high def main( @@ -188,7 +170,7 @@ def main( ) write_runtime_status(stage_name, phase="optimization_complete", state="running") - allocation = np.array([solution["allocation"][i] for i in range(n)], dtype=float) + allocation = solution_allocation_vector(solution, n) loan_amounts = ( test_sample["loan_amnt"].to_numpy(dtype=float) if "loan_amnt" in test_sample.columns diff --git a/scripts/optimize_portfolio_tradeoff.py b/scripts/optimize_portfolio_tradeoff.py index 7927d62..8a2df6b 100644 --- a/scripts/optimize_portfolio_tradeoff.py +++ b/scripts/optimize_portfolio_tradeoff.py @@ -22,10 +22,8 @@ from loguru import logger from src.models.conformal_artifacts import load_conformal_intervals -from src.optimization.portfolio_model import ( - compute_effective_pd, - optimize_portfolio_allocation, -) +from src.optimization.input_alignment import align_candidate_intervals +from src.optimization.policy_evaluation import solve_policy_allocation from src.utils.artifact_metadata import resolve_run_tag from src.utils.pipeline_runtime import ( atomic_write_json, @@ -99,69 +97,27 @@ def _align_loans_and_intervals( random_state: int, ) -> tuple[pd.DataFrame, pd.DataFrame]: """Align candidate loans with interval rows by id where possible.""" - max_candidates_norm = None if int(max_candidates) <= 0 else int(max_candidates) - if "id" in candidates.columns and "id" in intervals.columns: - cand = candidates.copy() - ints = intervals.copy() - cand["_id_join"] = cand["id"].astype(str) - ints["_id_join"] = ints["id"].astype(str) - ints = ints.drop_duplicates(subset="_id_join", keep="first") - merged = cand.merge(ints, on="_id_join", how="inner", suffixes=("", "_int")) - if merged.empty: - raise ValueError("ID-based merge between candidates and intervals returned zero rows.") - - n = len(merged) if max_candidates_norm is None else min(len(merged), max_candidates_norm) - if len(merged) > n: - idx = np.random.default_rng(random_state).choice( - np.arange(len(merged)), size=n, replace=False - ) - idx = np.sort(idx) - merged = merged.iloc[idx].reset_index(drop=True) - else: - merged = merged.reset_index(drop=True) - - loans = merged[candidates.columns].copy() - interval_cols = [c for c in intervals.columns if c in merged.columns] - ints_aligned = merged[interval_cols].copy() - logger.info( - f"Aligned tradeoff candidates and intervals by id: n={len(loans):,} " - f"(candidate_rows={len(candidates):,}, interval_rows={len(intervals):,})" - ) - return loans, ints_aligned - - if "_row_number" in intervals.columns: - cand = candidates.copy() - ints = intervals.copy() - cand["_row_number"] = np.arange(len(cand)) - merged = cand.merge(ints, on="_row_number", how="inner", suffixes=("", "_int")) - assert len(merged) == len(cand), ( - f"_row_number merge size mismatch: {len(merged)} != {len(cand)}" + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=max_candidates, + random_state=random_state, + ) + if aligned.mode == "position": + logger.warning( + "Conformal interval artifact has no id or _row_number alignment key; " + "using reproducible positional sampling in tradeoff analysis." ) - n = len(merged) if max_candidates_norm is None else min(len(merged), max_candidates_norm) - if len(merged) > n: - idx = np.random.default_rng(random_state).choice( - np.arange(len(merged)), size=n, replace=False - ) - merged = merged.iloc[np.sort(idx)].reset_index(drop=True) - else: - merged = merged.reset_index(drop=True) - loans = merged[candidates.columns].copy() - interval_cols = [c for c in intervals.columns if c in merged.columns] - ints_aligned = merged[interval_cols].copy() - logger.info(f"Aligned tradeoff candidates and intervals by _row_number: n={len(loans):,}") - return loans, ints_aligned - - logger.warning( - "Conformal interval artifact has no id or _row_number alignment key; using positional fallback in tradeoff analysis." + logger.info( + "Aligned tradeoff candidates and intervals by {}: n={:,} " + "(alignable_rows={:,}, candidate_rows={:,}, interval_rows={:,})", + aligned.mode, + aligned.selected_rows, + aligned.available_rows, + len(candidates), + len(intervals), ) - n = min(len(candidates), len(intervals)) - if max_candidates_norm is not None: - n = min(n, max_candidates_norm) - idx = np.random.default_rng(random_state).choice(np.arange(n), size=n, replace=False) - idx = np.sort(idx) - loans = candidates.iloc[idx].reset_index(drop=True).copy() - ints_aligned = intervals.iloc[idx].reset_index(drop=True).copy() - return loans, ints_aligned + return aligned.candidates, aligned.intervals def _write_candidate_universe(loans: pd.DataFrame, *, path: str, run_tag: str) -> None: @@ -327,37 +283,7 @@ def _solve_single( cuopt_presolve: int | None = 1, cuopt_parameters: dict[str, Any] | None = None, ) -> tuple[dict[str, float | int | str], np.ndarray]: - segment_labels: np.ndarray | None = None - if policy_mode in { - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - }: - grade = ( - loans["grade"].fillna("unknown").astype(str) - if "grade" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - term = ( - loans["term"].fillna("unknown").astype(str) - if "term" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - verification = ( - loans["verification_status"].fillna("unknown").astype(str) - if "verification_status" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - segment_labels = (grade + "|" + term + "|" + verification).to_numpy(dtype=object) - pd_constraint = compute_effective_pd( - pd_point=pd_point, - pd_high=pd_high, - policy_mode=policy_mode, - gamma=gamma, - delta_cap_quantile=delta_cap_quantile, - tail_focus_quantile=tail_focus_quantile, - segment_labels=segment_labels, - ) - solution = optimize_portfolio_allocation( + result = solve_policy_allocation( loans=loans, pd_point=pd_point, pd_low=pd_low, @@ -366,12 +292,15 @@ def _solve_single( int_rates=int_rates, total_budget=total_budget, max_concentration=max_concentration, - max_portfolio_pd=risk_tolerance, + risk_tolerance=risk_tolerance, robust=robust, uncertainty_aversion=uncertainty_aversion, min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, - pd_constraint_override=pd_constraint, + policy_mode=policy_mode, + gamma=gamma, + delta_cap_quantile=delta_cap_quantile, + tail_focus_quantile=tail_focus_quantile, time_limit=time_limit, threads=threads, solver_backend=solver_backend, @@ -379,9 +308,9 @@ def _solve_single( cuopt_presolve=cuopt_presolve, cuopt_parameters=cuopt_parameters, ) - + solution = result.solution n = len(loans) - allocation = np.array([solution["allocation"][i] for i in range(n)], dtype=float) + allocation = result.allocation loan_amounts = ( loans["loan_amnt"].to_numpy(dtype=float) if "loan_amnt" in loans.columns @@ -394,7 +323,7 @@ def _solve_single( expected_return = float(np.sum(allocation * loan_amounts * int_rates)) economic_return = expected_return - expected_loss realized_total_return = _compute_realized_total_return( - solution["allocation"], + allocation, loan_amounts, int_rates, default_flag if default_flag is not None else np.zeros(n, dtype=int), @@ -410,10 +339,10 @@ def _solve_single( return { "solver_status": str(solution["solver_status"]), "solver_backend": str(solver_backend), - "policy_mode": str(policy_mode), - "gamma": float(gamma), - "delta_cap_quantile": float(delta_cap_quantile), - "tail_focus_quantile": float(tail_focus_quantile), + "policy_mode": result.policy_mode.value, + "gamma": result.gamma, + "delta_cap_quantile": result.delta_cap_quantile, + "tail_focus_quantile": result.tail_focus_quantile, "objective_value": float(solution["objective_value"]), "n_funded": int(solution["n_funded"]), "total_allocated": total_allocated, @@ -430,23 +359,16 @@ def _solve_single( def _compute_realized_total_return( - allocation: dict[int, float], + allocation: np.ndarray, loan_amounts: np.ndarray, int_rates: np.ndarray, default_flag: np.ndarray, *, lgd: float = 0.45, ) -> float: - total = 0.0 - for i in range(len(loan_amounts)): - alloc = float(allocation.get(i, 0.0)) - if alloc <= 0.01: - continue - if int(default_flag[i]) == 1: - total += alloc * float(loan_amounts[i]) * (-lgd) - else: - total += alloc * float(loan_amounts[i]) * float(int_rates[i]) - return float(total) + funded = allocation > 0.01 + realized_rate = np.where(default_flag.astype(int) == 1, -float(lgd), int_rates) + return float(np.sum(allocation[funded] * loan_amounts[funded] * realized_rate[funded])) def _allocation_similarity(a: np.ndarray, b: np.ndarray) -> float: diff --git a/scripts/run_comparison.py b/scripts/run_comparison.py index 02135eb..c9eea75 100644 --- a/scripts/run_comparison.py +++ b/scripts/run_comparison.py @@ -310,6 +310,15 @@ class GateResult: details: dict[str, Any] +@dataclass +class StatusMetadataObservation: + row: dict[str, Any] + run_tag: str | None + generated_at: datetime | None + missing_metadata: bool + mismatched_run_tag: bool + + def _load_fairness_policy_contract(config_path: Path = FAIRNESS_POLICY_PATH) -> dict[str, Any]: """Return fairness business threshold contract from policy config.""" if not config_path.exists(): @@ -358,9 +367,7 @@ def _parse_iso_datetime(value: Any) -> datetime | None: return None -def _collect_status_metadata( - cur_metrics: dict[str, Any], *, expected_run_tag: str -) -> dict[str, Any]: +def _status_metadata_sources(cur_metrics: dict[str, Any]) -> dict[str, Any]: sources = { "reports/dvc/metrics_summary.json": cur_metrics.get("dvc_metrics_meta", {}), "data/processed/pipeline_summary.json": cur_metrics.get("pipeline_summary", {}), @@ -379,80 +386,140 @@ def _collect_status_metadata( for artifact_name, payload in optional_sources.items(): if isinstance(payload, dict) and payload: sources[artifact_name] = payload - rows: list[dict[str, Any]] = [] - run_tags: list[str] = [] - generated_times: list[datetime] = [] - missing_metadata_artifacts: list[str] = [] - mismatched_run_tag_artifacts: list[str] = [] - - for artifact_name, payload in sources.items(): - payload = payload if isinstance(payload, dict) else {} - schema_version = payload.get("schema_version") - generated_at_utc = payload.get("generated_at_utc") - run_tag = payload.get("run_tag") - missing_fields = [ - key - for key, value in { - "schema_version": schema_version, - "generated_at_utc": generated_at_utc, - "run_tag": run_tag, - }.items() - if value in (None, "", []) - ] - parsed_dt = _parse_iso_datetime(generated_at_utc) - if run_tag not in (None, "", []): - run_tags.append(str(run_tag)) - if parsed_dt is not None: - generated_times.append(parsed_dt) - if missing_fields: - missing_metadata_artifacts.append(artifact_name) - if run_tag not in (None, "", []) and str(run_tag) != expected_run_tag: - mismatched_run_tag_artifacts.append(artifact_name) - rows.append( - { - "artifact": artifact_name, - "schema_version": schema_version, - "generated_at_utc": generated_at_utc, - "run_tag": run_tag, - "missing_metadata_fields": missing_fields, - } - ) + return sources + + +def _missing_status_fields( + *, + schema_version: Any, + generated_at_utc: Any, + run_tag: Any, +) -> list[str]: + return [ + key + for key, value in { + "schema_version": schema_version, + "generated_at_utc": generated_at_utc, + "run_tag": run_tag, + }.items() + if value in (None, "", []) + ] + + +def _status_metadata_observation( + artifact_name: str, + payload: Any, + *, + expected_run_tag: str, +) -> StatusMetadataObservation: + payload = payload if isinstance(payload, dict) else {} + schema_version = payload.get("schema_version") + generated_at_utc = payload.get("generated_at_utc") + run_tag = payload.get("run_tag") + missing_fields = _missing_status_fields( + schema_version=schema_version, + generated_at_utc=generated_at_utc, + run_tag=run_tag, + ) + run_tag_text = None if run_tag in (None, "", []) else str(run_tag) + return StatusMetadataObservation( + row={ + "artifact": artifact_name, + "schema_version": schema_version, + "generated_at_utc": generated_at_utc, + "run_tag": run_tag, + "missing_metadata_fields": missing_fields, + }, + run_tag=run_tag_text, + generated_at=_parse_iso_datetime(generated_at_utc), + missing_metadata=bool(missing_fields), + mismatched_run_tag=run_tag_text is not None and run_tag_text != expected_run_tag, + ) + +def _timestamp_skew_seconds(generated_times: list[datetime]) -> float | None: + if len(generated_times) < 2: + return None + return float((max(generated_times) - min(generated_times)).total_seconds()) + + +def _run_tag_coherence( + *, + run_tags: list[str], + mismatched_artifacts: list[str], + expected_run_tag: str, +) -> dict[str, Any]: unique_run_tags = sorted(set(run_tags)) run_tag_consistent = len(unique_run_tags) == 1 run_tag_matches_expected = run_tag_consistent and unique_run_tags == [expected_run_tag] - timestamp_skew_seconds = None - if len(generated_times) >= 2: - timestamp_skew_seconds = float( - (max(generated_times) - min(generated_times)).total_seconds() + non_causal_mismatches = [ + artifact for artifact in mismatched_artifacts if artifact not in _CAUSAL_INSIGHTS_ARTIFACTS + ] + causal_only_mismatch = bool(mismatched_artifacts) and len(non_causal_mismatches) == 0 + return { + "run_tags_observed": unique_run_tags, + "run_tag_consistent": run_tag_consistent, + "run_tag_matches_expected": run_tag_matches_expected, + "run_tag_matches_expected_operational": run_tag_matches_expected or causal_only_mismatch, + "causal_only_mismatch": causal_only_mismatch, + "non_causal_mismatched_run_tag_artifacts": non_causal_mismatches, + } + + +def _collect_status_metadata( + cur_metrics: dict[str, Any], *, expected_run_tag: str +) -> dict[str, Any]: + observations = [ + _status_metadata_observation( + artifact_name, + payload, + expected_run_tag=expected_run_tag, ) + for artifact_name, payload in _status_metadata_sources(cur_metrics).items() + ] + rows = [observation.row for observation in observations] + run_tags = [ + observation.run_tag for observation in observations if observation.run_tag is not None + ] + generated_times = [ + observation.generated_at + for observation in observations + if observation.generated_at is not None + ] + missing_metadata_artifacts = [ + str(observation.row["artifact"]) + for observation in observations + if observation.missing_metadata + ] + mismatched_run_tag_artifacts = [ + str(observation.row["artifact"]) + for observation in observations + if observation.mismatched_run_tag + ] + + tag_coherence = _run_tag_coherence( + run_tags=run_tags, + mismatched_artifacts=mismatched_run_tag_artifacts, + expected_run_tag=expected_run_tag, + ) + timestamp_skew_seconds = _timestamp_skew_seconds(generated_times) timestamp_coherent = timestamp_skew_seconds is None or timestamp_skew_seconds <= float( COHERENCE_TIMESTAMP_MAX_SKEW_SECONDS ) all_have_metadata = len(missing_metadata_artifacts) == 0 - - # Allow mismatches that are exclusively causal/CATE insights_only artifacts — - # these are documented as not-regenerated in every run by design. - non_causal_mismatches = [ - a for a in mismatched_run_tag_artifacts if a not in _CAUSAL_INSIGHTS_ARTIFACTS - ] - causal_only_mismatch = bool(mismatched_run_tag_artifacts) and len(non_causal_mismatches) == 0 - run_tag_matches_expected_operational = run_tag_matches_expected or causal_only_mismatch - - passed = bool(all_have_metadata and run_tag_matches_expected_operational and timestamp_coherent) + passed = bool( + all_have_metadata + and tag_coherence["run_tag_matches_expected_operational"] + and timestamp_coherent + ) return { "expected_run_tag": expected_run_tag, "critical_artifacts": rows, "all_have_metadata": all_have_metadata, "missing_metadata_artifacts": missing_metadata_artifacts, - "run_tags_observed": unique_run_tags, - "run_tag_consistent": run_tag_consistent, - "run_tag_matches_expected": run_tag_matches_expected, - "run_tag_matches_expected_operational": run_tag_matches_expected_operational, - "causal_only_mismatch": causal_only_mismatch, + **tag_coherence, "mismatched_run_tag_artifacts": mismatched_run_tag_artifacts, - "non_causal_mismatched_run_tag_artifacts": non_causal_mismatches, "timestamp_skew_seconds": timestamp_skew_seconds, "timestamp_coherent": bool(timestamp_coherent), "timestamp_max_skew_seconds": int(COHERENCE_TIMESTAMP_MAX_SKEW_SECONDS), @@ -661,43 +728,104 @@ def _gate_conformal(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: ) -def _gate_ab_no_regression(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: - b = base.get("ab_simulation_status", {}) - c = cur.get("ab_simulation_status", {}) +def _ab_total_returns(status: dict[str, Any]) -> tuple[float, float]: + return ( + _safe_float((status.get("metrics_a") or {}).get("total_return")), + _safe_float((status.get("metrics_b") or {}).get("total_return")), + ) - b_a = _safe_float((b.get("metrics_a") or {}).get("total_return")) - b_b = _safe_float((b.get("metrics_b") or {}).get("total_return")) - c_a = _safe_float((c.get("metrics_a") or {}).get("total_return")) - c_b = _safe_float((c.get("metrics_b") or {}).get("total_return")) - no_reg = c.get("no_regression", {}) if isinstance(c.get("no_regression"), dict) else {} +def _ab_current_no_regression( + status: dict[str, Any], + control_return: float, + robust_return: float, +) -> tuple[bool, float, float, dict[str, Any]]: + no_reg = ( + status.get("no_regression", {}) if isinstance(status.get("no_regression"), dict) else {} + ) cross_gate = ( - c.get("cross_scenario_gate", {}) if isinstance(c.get("cross_scenario_gate"), dict) else {} + status.get("cross_scenario_gate", {}) + if isinstance(status.get("cross_scenario_gate"), dict) + else {} ) - c_diff = _safe_float( + diff = _safe_float( no_reg.get("diff_total_return"), - default=(c_b - c_a if np.isfinite(c_a) and np.isfinite(c_b) else float("nan")), + default=( + robust_return - control_return + if np.isfinite(control_return) and np.isfinite(robust_return) + else float("nan") + ), ) - c_tol = _safe_float( + tolerance = _safe_float( no_reg.get("tolerance_total_return"), - default=(abs(c_a) * 0.05 if np.isfinite(c_a) else float("nan")), + default=(abs(control_return) * 0.05 if np.isfinite(control_return) else float("nan")), ) - - self_no_reg_ok = ( + passed = ( bool(no_reg.get("passed")) if "passed" in no_reg - else (np.isnan(c_diff) or np.isnan(c_tol) or (c_diff >= -c_tol)) + else (np.isnan(diff) or np.isnan(tolerance) or (diff >= -tolerance)) ) - if str(c.get("decision_scenario", "")).strip() == "selective_ambiguity_defer" and bool( + if str(status.get("decision_scenario", "")).strip() == "selective_ambiguity_defer" and bool( cross_gate.get("passed", False) ): - self_no_reg_ok = True + passed = True + return bool(passed), float(diff), float(tolerance), cross_gate + + +def _ab_baseline_checks( + *, + baseline_control_return: float, + baseline_robust_return: float, + current_control_return: float, + current_robust_return: float, + current_diff: float, +) -> tuple[dict[str, bool], float]: + baseline_diff = ( + baseline_robust_return - baseline_control_return + if np.isfinite(baseline_control_return) and np.isfinite(baseline_robust_return) + else float("nan") + ) + baseline_tol = ( + abs(baseline_control_return) * 0.05 + if np.isfinite(baseline_control_return) + else float("nan") + ) + return ( + { + "control_vs_baseline_ok": bool( + np.isnan(baseline_control_return) + or np.isnan(current_control_return) + or (current_control_return >= baseline_control_return - baseline_tol) + ), + "robust_vs_baseline_ok": bool( + np.isnan(baseline_robust_return) + or np.isnan(current_robust_return) + or (current_robust_return >= baseline_robust_return - baseline_tol) + ), + "gap_vs_baseline_ok": bool( + np.isnan(baseline_diff) + or np.isnan(current_diff) + or (current_diff >= baseline_diff - baseline_tol) + ), + }, + float(baseline_diff), + ) - b_diff = b_b - b_a if np.isfinite(b_a) and np.isfinite(b_b) else float("nan") - baseline_tol = abs(b_a) * 0.05 if np.isfinite(b_a) else float("nan") - control_vs_baseline_ok = np.isnan(b_a) or np.isnan(c_a) or (c_a >= b_a - baseline_tol) - robust_vs_baseline_ok = np.isnan(b_b) or np.isnan(c_b) or (c_b >= b_b - baseline_tol) - gap_vs_baseline_ok = np.isnan(b_diff) or np.isnan(c_diff) or (c_diff >= b_diff - baseline_tol) + +def _gate_ab_no_regression(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: + b = base.get("ab_simulation_status", {}) + c = cur.get("ab_simulation_status", {}) + + b_a, b_b = _ab_total_returns(b) + c_a, c_b = _ab_total_returns(c) + self_no_reg_ok, c_diff, c_tol, cross_gate = _ab_current_no_regression(c, c_a, c_b) + baseline_checks, b_diff = _ab_baseline_checks( + baseline_control_return=b_a, + baseline_robust_return=b_b, + current_control_return=c_a, + current_robust_return=c_b, + current_diff=c_diff, + ) passed = bool(self_no_reg_ok) @@ -709,14 +837,12 @@ def _gate_ab_no_regression(base: dict[str, Any], cur: dict[str, Any]) -> GateRes "checks": { "self_no_regression_ok": bool(self_no_reg_ok), "cross_scenario_gate_ok": bool(cross_gate.get("passed", False)), - "control_vs_baseline_ok": bool(control_vs_baseline_ok), - "robust_vs_baseline_ok": bool(robust_vs_baseline_ok), - "gap_vs_baseline_ok": bool(gap_vs_baseline_ok), + **baseline_checks, }, "warnings": { - "control_vs_baseline_warning": bool(not control_vs_baseline_ok), - "robust_vs_baseline_warning": bool(not robust_vs_baseline_ok), - "gap_vs_baseline_warning": bool(not gap_vs_baseline_ok), + "control_vs_baseline_warning": bool(not baseline_checks["control_vs_baseline_ok"]), + "robust_vs_baseline_warning": bool(not baseline_checks["robust_vs_baseline_ok"]), + "gap_vs_baseline_warning": bool(not baseline_checks["gap_vs_baseline_ok"]), }, "current": { "control_total_return": c_a, @@ -930,81 +1056,104 @@ def _write_snapshot(run_tag: str) -> Path: return path -def _write_compare(run_tag: str, baseline_path: Path) -> tuple[Path, Path]: - baseline_path = baseline_path.expanduser().resolve() - baseline = json.loads(baseline_path.read_text(encoding="utf-8")) - current = _snapshot_payload(run_tag) - gate_results = [ - _gate_artifact_coherence(current["metrics"], run_tag), - _gate_semantic_coherence(current["metrics"]), - _gate_pd(baseline["metrics"], current["metrics"]), - _gate_conformal(baseline["metrics"], current["metrics"]), - _gate_ab_no_regression(baseline["metrics"], current["metrics"]), - _gate_fairness(baseline["metrics"], current["metrics"]), - _gate_fairness_absolute_business(baseline["metrics"], current["metrics"]), - _gate_survival(baseline["metrics"], current["metrics"]), - _gate_exports(current), +def _comparison_gate_results( + *, + baseline_metrics: dict[str, Any], + current_metrics: dict[str, Any], + current_snapshot: dict[str, Any], + run_tag: str, +) -> list[GateResult]: + return [ + _gate_artifact_coherence(current_metrics, run_tag), + _gate_semantic_coherence(current_metrics), + _gate_pd(baseline_metrics, current_metrics), + _gate_conformal(baseline_metrics, current_metrics), + _gate_ab_no_regression(baseline_metrics, current_metrics), + _gate_fairness(baseline_metrics, current_metrics), + _gate_fairness_absolute_business(baseline_metrics, current_metrics), + _gate_survival(baseline_metrics, current_metrics), + _gate_exports(current_snapshot), ] - conformal_gate = next((g for g in gate_results if g.name == "conformal_policy"), None) - ab_gate = next((g for g in gate_results if g.name == "ab_no_regression"), None) - coherence_gate = next((g for g in gate_results if g.name == "artifact_coherence"), None) - semantic_gate = next((g for g in gate_results if g.name == "semantic_coherence"), None) - conformal_details = conformal_gate.details if conformal_gate is not None else {} - ab_details = ab_gate.details if ab_gate is not None else {} - fairness_abs_gate = next( - (g for g in gate_results if g.name == "fairness_absolute_business"), None - ) - conformal_checks = conformal_details.get("checks", {}) + + +def _gate_lookup(gate_results: list[GateResult]) -> dict[str, GateResult]: + return {gate.name: gate for gate in gate_results} + + +def _gate_pass(gates: dict[str, GateResult], name: str) -> bool: + gate = gates.get(name) + return bool(gate.passed) if gate is not None else False + + +def _gate_details(gates: dict[str, GateResult], name: str) -> dict[str, Any]: + gate = gates.get(name) + return gate.details if gate is not None else {} + + +def _comparison_gate_fields(gate_results: list[GateResult]) -> dict[str, Any]: + gates = _gate_lookup(gate_results) + conformal_details = _gate_details(gates, "conformal_policy") conformal_diagnostics = conformal_details.get("diagnostics", {}) - ab_diagnostics = ab_details.get("diagnostics", {}) - coherence_details = coherence_gate.details if coherence_gate is not None else {} - try: - baseline_path_out = str(baseline_path.relative_to(ROOT)) - except ValueError: - baseline_path_out = str(baseline_path) - report = { - "schema_version": SCHEMA_VERSION, - "run_tag": run_tag, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "baseline_path": baseline_path_out, - "overall_pass": bool(all(g.passed for g in gate_results)), + ab_diagnostics = _gate_details(gates, "ab_no_regression").get("diagnostics", {}) + return { + "overall_pass": bool(all(gate.passed for gate in gate_results)), "operational_overall_pass": bool( - all(g.passed for g in gate_results if g.name in _OPERATIONAL_GATE_NAMES) + all(gate.passed for gate in gate_results if gate.name in _OPERATIONAL_GATE_NAMES) + ), + "artifact_coherence_pass": _gate_pass(gates, "artifact_coherence"), + "artifact_coherence": _gate_details(gates, "artifact_coherence"), + "semantic_coherence_pass": _gate_pass(gates, "semantic_coherence"), + "semantic_coherence": _gate_details(gates, "semantic_coherence"), + "conformal_promotion_pass": bool( + conformal_details.get("checks", {}).get("conformal_promotion_pass", False) ), - "artifact_coherence_pass": bool(coherence_gate.passed) - if coherence_gate is not None - else False, - "artifact_coherence": coherence_details, - "semantic_coherence_pass": bool(semantic_gate.passed) - if semantic_gate is not None - else False, - "semantic_coherence": semantic_gate.details if semantic_gate is not None else {}, - "conformal_promotion_pass": bool(conformal_checks.get("conformal_promotion_pass", False)), "conformal_retired_backtest_checks": conformal_diagnostics.get( "retired_backtest_checks", [] ), - "ab_no_regression_pass": bool(ab_gate.passed) if ab_gate is not None else False, - "fairness_absolute_business_pass": bool(fairness_abs_gate.passed) - if fairness_abs_gate is not None - else False, + "ab_no_regression_pass": _gate_pass(gates, "ab_no_regression"), + "fairness_absolute_business_pass": _gate_pass(gates, "fairness_absolute_business"), "ab_gate_mode": str(ab_diagnostics.get("gate_mode", "no_regression")), "ab_significant": bool(ab_diagnostics.get("significant", False)), "ab_significance_role": str(ab_diagnostics.get("significance_role", "diagnostic")), + } + + +def _comparison_quality_contract() -> dict[str, Any]: + return { + "conformal_checks_required": 13, + "ab_gate_mode": "no_regression", + "ab_significance_role": "diagnostic", + "fairness_gates": ["fairness_relative", "fairness_absolute_business"], + "fairness_policy_path": _path_for_report(FAIRNESS_POLICY_PATH), + "artifact_coherence_required": True, + "semantic_coherence_required": True, + "required_status_metadata": ["schema_version", "generated_at_utc", "run_tag"], + } + + +def _comparison_report( + *, + run_tag: str, + baseline_path: Path, + baseline: dict[str, Any], + current: dict[str, Any], + gate_results: list[GateResult], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "run_tag": run_tag, + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "baseline_path": _path_for_report(baseline_path), + **_comparison_gate_fields(gate_results), "gates": [{"name": g.name, "passed": g.passed, "details": g.details} for g in gate_results], "artifact_changes": _compare_artifacts(baseline, current), "baseline_head": baseline.get("git", {}).get("head", ""), "current_head": current.get("git", {}).get("head", ""), - "quality_contract": { - "conformal_checks_required": 13, - "ab_gate_mode": "no_regression", - "ab_significance_role": "diagnostic", - "fairness_gates": ["fairness_relative", "fairness_absolute_business"], - "fairness_policy_path": _path_for_report(FAIRNESS_POLICY_PATH), - "artifact_coherence_required": True, - "semantic_coherence_required": True, - "required_status_metadata": ["schema_version", "generated_at_utc", "run_tag"], - }, + "quality_contract": _comparison_quality_contract(), } + + +def _write_comparison_files(run_tag: str, report: dict[str, Any]) -> tuple[Path, Path]: out_dir = OUT_ROOT / run_tag out_dir.mkdir(parents=True, exist_ok=True) json_path = out_dir / "comparison.json" @@ -1019,6 +1168,26 @@ def _write_compare(run_tag: str, baseline_path: Path) -> tuple[Path, Path]: return json_path, md_path +def _write_compare(run_tag: str, baseline_path: Path) -> tuple[Path, Path]: + baseline_path = baseline_path.expanduser().resolve() + baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + current = _snapshot_payload(run_tag) + gate_results = _comparison_gate_results( + baseline_metrics=baseline["metrics"], + current_metrics=current["metrics"], + current_snapshot=current, + run_tag=run_tag, + ) + report = _comparison_report( + run_tag=run_tag, + baseline_path=baseline_path, + baseline=baseline, + current=current, + gate_results=gate_results, + ) + return _write_comparison_files(run_tag, report) + + def main() -> None: parser = argparse.ArgumentParser(description="Snapshot/compare run artifacts with gates.") sub = parser.add_subparsers(dest="cmd", required=True) diff --git a/scripts/run_cqr_comparison.py b/scripts/run_cqr_comparison.py index 08df8d1..409d1e1 100644 --- a/scripts/run_cqr_comparison.py +++ b/scripts/run_cqr_comparison.py @@ -188,7 +188,7 @@ def make_qreg(quantile: float) -> CatBoostRegressor: # MAPIE CQR: pass list [alpha/2, 1-alpha/2, 0.5] in that order cqr = ConformalizedQuantileRegressor( - estimator=[qreg_lo, qreg_hi, qreg_mid], + estimator=cast(Any, [qreg_lo, qreg_hi, qreg_mid]), confidence_level=1.0 - alpha, prefit=True, ) diff --git a/scripts/run_crpto_vs_spo_stability.py b/scripts/run_crpto_vs_spo_stability.py index aebc3be..b97729c 100644 --- a/scripts/run_crpto_vs_spo_stability.py +++ b/scripts/run_crpto_vs_spo_stability.py @@ -21,10 +21,12 @@ from __future__ import annotations import argparse +import importlib import json import shutil import time from collections import OrderedDict +from numbers import Real from pathlib import Path from typing import Any @@ -34,25 +36,31 @@ import pandas as pd from loguru import logger -from scripts.run_spo_real import ( - LGD, - NUMERIC_FEATURES, - RANDOM_SEED, - CreditPortfolioLP, - _compute_regret, - _compute_true_optima, - _index_costs, - _load_pd_artifacts, - _predict_calibrated_costs, - _prep_features, - _sample_instances, - _train_spo, -) from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag matplotlib.use("Agg") SCHEMA_VERSION = "2026-03-22.1" +LGD = 0.40 +RANDOM_SEED = 42 + +NUMERIC_FEATURES = [ + "loan_amnt", + "int_rate", + "annual_inc", + "dti", + "fico_range_low", + "open_acc", + "revol_bal", + "revol_util", + "total_acc", + "installment", + "emp_length", + "pub_rec", + "delinq_2yrs", + "inq_last_6mths", + "mths_since_last_delinq", +] REPO_ROOT = Path(__file__).resolve().parents[1] DATA_DIR = REPO_ROOT / "data" / "processed" @@ -108,9 +116,31 @@ ) +def _require_optional_module(module_name: str) -> Any: + """Import an optional SPO dependency with an explicit experiment-level error.""" + try: + return importlib.import_module(module_name) + except ImportError as exc: + raise RuntimeError( + "SPO stability is an optional experiment. Install the `spo` extras " + f"before running this script; missing module: {module_name}." + ) from exc + + # ── Period assignment ──────────────────────────────────────────────────────── +def _load_spo_real_module() -> Any: + """Load SPO helpers lazily so this module remains importable without PyEPO.""" + try: + return importlib.import_module("scripts.run_spo_real") + except RuntimeError as exc: + raise RuntimeError( + "SPO stability is an optional experiment. Install the `spo` extras " + "before running this script." + ) from exc + + def _assign_periods(issue_d: pd.Series) -> pd.Series: """Assign each loan to a temporal period based on issue_d.""" dt = pd.to_datetime(issue_d) @@ -160,6 +190,219 @@ def _evaluate_period_coverage(ci_slice: pd.DataFrame) -> dict: # ── Figure generation ──────────────────────────────────────────────────────── +def _available_numeric_features(train: pd.DataFrame, test: pd.DataFrame) -> list[str]: + return [ + feature + for feature in NUMERIC_FEATURES + if feature in train.columns and feature in test.columns + ] + + +def _period_masks(test_periods: pd.Series) -> dict[str, np.ndarray]: + return {name: (test_periods.values == name) for name in PERIODS} + + +def _period_default_rate(test: pd.DataFrame, mask: np.ndarray) -> float: + n_loans = int(mask.sum()) + return float(test.loc[mask, "default_flag"].mean()) if n_loans > 0 else 0.0 + + +def _init_period_regrets() -> dict[str, dict[str, list[float]]]: + return {name: {"two_stage": [], "spo_plus": [], "conformal_robust": []} for name in PERIODS} + + +def _period_sample_seed(seed: int, period_name: str) -> int: + """Stable per-period seed; avoids Python's process-randomized hash().""" + period_offset = list(PERIODS).index(period_name) + 1 + return int(seed + period_offset * 100_000) + + +def _period_test_instance_count(n_period: int, n_items: int) -> int: + return max(min(80, n_period // n_items), 5) + + +def _valid_regret_values(values: list[float]) -> list[float]: + return [value for value in values if not np.isnan(value)] + + +def _mean_std(values: list[float]) -> tuple[float, float]: + valid = _valid_regret_values(values) + if not valid: + return float("nan"), float("nan") + return float(np.mean(valid)), float(np.std(valid)) + + +def _spo_improvement_pct(two_stage_mean: float, spo_mean: float, has_values: bool) -> float | None: + if not has_values: + return None + return (two_stage_mean - spo_mean) / (abs(two_stage_mean) + 1e-9) * 100 + + +def _coverage_by_period( + ci: pd.DataFrame, + period_masks: dict[str, np.ndarray], +) -> dict[str, dict[str, Any]]: + period_coverage: dict[str, dict[str, Any]] = {} + for period_name, mask in period_masks.items(): + ci_slice = ci.loc[mask] + if len(ci_slice) == 0: + continue + period_coverage[period_name] = _evaluate_period_coverage(ci_slice) + logger.info( + " {} coverage: 90%={:.2%} width={:.4f} min_grade={:.2%}", + period_name, + period_coverage[period_name]["coverage_90"], + period_coverage[period_name]["avg_width_90"], + period_coverage[period_name].get("min_grade_coverage_90", 0) or 0, + ) + return period_coverage + + +def _detail_rows( + *, + test: pd.DataFrame, + period_masks: dict[str, np.ndarray], + per_period_regrets: dict[str, dict[str, list[float]]], + period_coverage: dict[str, dict[str, Any]], +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for period_name in PERIODS: + mask = period_masks[period_name] + n_loans = int(mask.sum()) + regrets = per_period_regrets[period_name] + cov = period_coverage.get(period_name, {}) + + ts_mean, ts_std = _mean_std(regrets["two_stage"]) + spo_mean, spo_std = _mean_std(regrets["spo_plus"]) + cr_mean, cr_std = _mean_std(regrets["conformal_robust"]) + has_spo_comparison = bool( + _valid_regret_values(regrets["two_stage"]) and _valid_regret_values(regrets["spo_plus"]) + ) + + rows.append( + { + "period": period_name, + "n_loans": n_loans, + "default_rate": _period_default_rate(test, mask), + "two_stage_mean_regret": ts_mean, + "two_stage_std_regret": ts_std, + "spo_plus_mean_regret": spo_mean, + "spo_plus_std_regret": spo_std, + "conformal_robust_mean_regret": cr_mean, + "conformal_robust_std_regret": cr_std, + "spo_improvement_pct": _spo_improvement_pct( + ts_mean, + spo_mean, + has_spo_comparison, + ), + "coverage_90": cov.get("coverage_90"), + "coverage_95": cov.get("coverage_95"), + "avg_width_90": cov.get("avg_width_90"), + "min_grade_coverage_90": cov.get("min_grade_coverage_90"), + } + ) + return rows + + +def _round_optional(value: object, digits: int) -> float | None: + if isinstance(value, Real): + return round(float(value), digits) + return None + + +def _period_summary_row( + row: dict[str, Any], + regrets: dict[str, list[float]], +) -> dict[str, Any]: + return { + "n_loans": int(row["n_loans"]), + "default_rate": round(float(row["default_rate"]), 4), + "regret": { + "two_stage": { + "mean": round(float(row["two_stage_mean_regret"]), 6), + "std": round(float(row["two_stage_std_regret"]), 6), + "per_seed": regrets["two_stage"], + }, + "spo_plus": { + "mean": round(float(row["spo_plus_mean_regret"]), 6), + "std": round(float(row["spo_plus_std_regret"]), 6), + "per_seed": regrets["spo_plus"], + }, + "conformal_robust": { + "mean": round(float(row["conformal_robust_mean_regret"]), 6), + "std": round(float(row["conformal_robust_std_regret"]), 6), + "per_seed": regrets["conformal_robust"], + }, + }, + "spo_improvement_vs_ts_pct": _round_optional(row["spo_improvement_pct"], 2), + "coverage_90": _round_optional(row["coverage_90"], 4), + "coverage_95": _round_optional(row["coverage_95"], 4), + "avg_width_90": _round_optional(row["avg_width_90"], 4), + "min_grade_coverage_90": _round_optional(row["min_grade_coverage_90"], 4), + } + + +def _per_period_json( + rows: list[dict[str, Any]], + per_period_regrets: dict[str, dict[str, list[float]]], +) -> dict[str, Any]: + return { + str(row["period"]): _period_summary_row( + row, + per_period_regrets[str(row["period"])], + ) + for row in rows + } + + +def _non_null_float_values(rows: list[dict[str, Any]], key: str) -> list[float]: + return [float(row[key]) for row in rows if row[key] is not None] + + +def _stability_summary(rows: list[dict[str, Any]]) -> dict[str, Any]: + coverages_90 = _non_null_float_values(rows, "coverage_90") + spo_improvements = _non_null_float_values(rows, "spo_improvement_pct") + return { + "coverage_always_above_target": all(c >= 0.90 for c in coverages_90), + "coverage_range": [round(min(coverages_90), 4), round(max(coverages_90), 4)], + "spo_improvement_range_pct": ( + [round(min(spo_improvements), 2), round(max(spo_improvements), 2)] + if spo_improvements + else None + ), + } + + +def _summary_payload( + *, + run_tag: str, + args: argparse.Namespace, + n_features: int, + feature_names: list[str], + rows: list[dict[str, Any]], + per_period_regrets: dict[str, dict[str, list[float]]], + total_time: float, +) -> dict[str, Any]: + return { + **build_artifact_metadata( + schema_version=SCHEMA_VERSION, run_tag=run_tag, allow_untracked=True + ), + "config": { + "n_items": args.n_items, + "budget": args.budget, + "n_train_instances": args.n_train, + "epochs": args.epochs, + "n_seeds": args.seeds, + "n_features": n_features, + "feature_names": feature_names, + "lgd": LGD, + }, + "per_period": _per_period_json(rows, per_period_regrets), + "stability_summary": _stability_summary(rows), + "train_time_seconds": round(total_time, 1), + } + + def _generate_stability_figure(detail_df: pd.DataFrame, out_dir: Path) -> None: """Two-panel figure: Panel A = regret by period, Panel B = coverage by period.""" fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(COL2, HEIGHT_M)) @@ -277,6 +520,17 @@ def main() -> int: ) # ── 1. Load data ──────────────────────────────────────────────────────── + spo_real = _load_spo_real_module() + CreditPortfolioLP = spo_real.CreditPortfolioLP + _compute_regret = spo_real._compute_regret + _compute_true_optima = spo_real._compute_true_optima + _index_costs = spo_real._index_costs + _load_pd_artifacts = spo_real._load_pd_artifacts + _predict_calibrated_costs = spo_real._predict_calibrated_costs + _prep_features = spo_real._prep_features + _sample_instances = spo_real._sample_instances + _train_spo = spo_real._train_spo + train = pd.read_parquet(DATA_DIR / "train_fe.parquet") test = pd.read_parquet(DATA_DIR / "test_fe.parquet") ci = pd.read_parquet(DATA_DIR / "conformal_intervals_mondrian.parquet") @@ -293,9 +547,7 @@ def main() -> int: logger.info("Period counts: {}", period_counts) # ── 2. Prepare features and costs ─────────────────────────────────────── - _tr_cols = train.columns - _te_cols = test.columns - avail = [f for f in NUMERIC_FEATURES if f in _tr_cols and f in _te_cols] + avail = _available_numeric_features(train, test) n_features = len(avail) logger.info("Using {} features: {}", n_features, avail) @@ -329,19 +581,15 @@ def main() -> int: c_robust_te_all = (pd_high * LGD - int_rate_te).astype(np.float32) # ── 3. Build period masks ─────────────────────────────────────────────── - period_masks = {} - for name in PERIODS: - mask = test_periods.values == name - period_masks[name] = mask + period_masks = _period_masks(test_periods) + for name, mask in period_masks.items(): n_loans = int(mask.sum()) - default_rate = float(test.loc[mask, "default_flag"].mean()) if n_loans > 0 else 0.0 + default_rate = _period_default_rate(test, mask) logger.info(" {} : {:,} loans, default rate {:.2%}", name, n_loans, default_rate) # ── 4. Multi-seed × multi-period evaluation ───────────────────────────── # Structure: per_period[period][method] = list of per-seed mean regrets - per_period_regrets: dict[str, dict[str, list[float]]] = { - name: {"two_stage": [], "spo_plus": [], "conformal_robust": []} for name in PERIODS - } + per_period_regrets = _init_period_regrets() t_total = time.time() @@ -379,9 +627,7 @@ def main() -> int: per_period_regrets[period_name][method].append(float("nan")) continue - n_test_period = min(80, n_period // args.n_items) - if n_test_period < 5: - n_test_period = 5 + n_test_period = _period_test_instance_count(n_period, args.n_items) # Slice arrays to this period period_idx = np.where(mask)[0] @@ -391,7 +637,7 @@ def main() -> int: c_robust_period = c_robust_te_all[period_idx] # Sample test instances within this period - rng_period = np.random.RandomState(seed + hash(period_name) % (2**31)) + rng_period = np.random.RandomState(_period_sample_seed(seed, period_name)) X_inst, c_inst, idx_inst = _sample_instances( X_period, c_period, args.n_items, n_test_period, rng_period ) @@ -399,8 +645,7 @@ def main() -> int: c_robust_inst = _index_costs(c_robust_period, idx_inst) # SPO+ predictions - import torch - + torch = _require_optional_module("torch") n_input = args.n_items * n_features X_flat = X_inst.reshape(n_test_period, n_input) with torch.no_grad(): @@ -433,63 +678,15 @@ def main() -> int: logger.info("All seeds done in {:.1f}s", total_time) # ── 5. Conformal coverage per period (deterministic, no seed) ─────────── - period_coverage: dict[str, dict] = {} - for period_name, mask in period_masks.items(): - ci_slice = ci.loc[mask] - if len(ci_slice) == 0: - continue - period_coverage[period_name] = _evaluate_period_coverage(ci_slice) - logger.info( - " {} coverage: 90%={:.2%} width={:.4f} min_grade={:.2%}", - period_name, - period_coverage[period_name]["coverage_90"], - period_coverage[period_name]["avg_width_90"], - period_coverage[period_name].get("min_grade_coverage_90", 0) or 0, - ) + period_coverage = _coverage_by_period(ci, period_masks) # ── 6. Aggregate into detail DataFrame ────────────────────────────────── - rows: list[dict[str, Any]] = [] - for period_name in PERIODS: - mask = period_masks[period_name] - n_loans = int(mask.sum()) - default_rate = float(test.loc[mask, "default_flag"].mean()) if n_loans > 0 else 0.0 - - regrets = per_period_regrets[period_name] - cov = period_coverage.get(period_name, {}) - - ts_vals = [v for v in regrets["two_stage"] if not np.isnan(v)] - spo_vals = [v for v in regrets["spo_plus"] if not np.isnan(v)] - cr_vals = [v for v in regrets["conformal_robust"] if not np.isnan(v)] - - ts_mean = float(np.mean(ts_vals)) if ts_vals else float("nan") - spo_mean = float(np.mean(spo_vals)) if spo_vals else float("nan") - cr_mean = float(np.mean(cr_vals)) if cr_vals else float("nan") - ts_std = float(np.std(ts_vals)) if ts_vals else float("nan") - spo_std = float(np.std(spo_vals)) if spo_vals else float("nan") - cr_std = float(np.std(cr_vals)) if cr_vals else float("nan") - - spo_improvement = ( - ((ts_mean - spo_mean) / (abs(ts_mean) + 1e-9) * 100) if ts_vals and spo_vals else None - ) - - rows.append( - { - "period": period_name, - "n_loans": n_loans, - "default_rate": default_rate, - "two_stage_mean_regret": ts_mean, - "two_stage_std_regret": ts_std, - "spo_plus_mean_regret": spo_mean, - "spo_plus_std_regret": spo_std, - "conformal_robust_mean_regret": cr_mean, - "conformal_robust_std_regret": cr_std, - "spo_improvement_pct": spo_improvement, - "coverage_90": cov.get("coverage_90"), - "coverage_95": cov.get("coverage_95"), - "avg_width_90": cov.get("avg_width_90"), - "min_grade_coverage_90": cov.get("min_grade_coverage_90"), - } - ) + rows = _detail_rows( + test=test, + period_masks=period_masks, + per_period_regrets=per_period_regrets, + period_coverage=period_coverage, + ) detail_df = pd.DataFrame(rows) detail_path = DATA_DIR / "crpto_vs_spo_stability_detail.parquet" @@ -497,82 +694,15 @@ def main() -> int: logger.info("Saved: {}", detail_path) # ── 7. Summary JSON ───────────────────────────────────────────────────── - coverages_90 = [float(r["coverage_90"]) for r in rows if r["coverage_90"] is not None] - spo_improvements = [ - float(r["spo_improvement_pct"]) for r in rows if r["spo_improvement_pct"] is not None - ] - - per_period_json: dict[str, Any] = {} - for r in rows: - period = str(r["period"]) - regrets = per_period_regrets[period] - per_period_json[period] = { - "n_loans": int(r["n_loans"]), - "default_rate": round(float(r["default_rate"]), 4), - "regret": { - "two_stage": { - "mean": round(float(r["two_stage_mean_regret"]), 6), - "std": round(float(r["two_stage_std_regret"]), 6), - "per_seed": regrets["two_stage"], - }, - "spo_plus": { - "mean": round(float(r["spo_plus_mean_regret"]), 6), - "std": round(float(r["spo_plus_std_regret"]), 6), - "per_seed": regrets["spo_plus"], - }, - "conformal_robust": { - "mean": round(float(r["conformal_robust_mean_regret"]), 6), - "std": round(float(r["conformal_robust_std_regret"]), 6), - "per_seed": regrets["conformal_robust"], - }, - }, - "spo_improvement_vs_ts_pct": ( - round(float(r["spo_improvement_pct"]), 2) - if r["spo_improvement_pct"] is not None - else None - ), - "coverage_90": ( - round(float(r["coverage_90"]), 4) if r["coverage_90"] is not None else None - ), - "coverage_95": ( - round(float(r["coverage_95"]), 4) if r["coverage_95"] is not None else None - ), - "avg_width_90": ( - round(float(r["avg_width_90"]), 4) if r["avg_width_90"] is not None else None - ), - "min_grade_coverage_90": ( - round(float(r["min_grade_coverage_90"]), 4) - if r["min_grade_coverage_90"] is not None - else None - ), - } - - summary = { - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, run_tag=run_tag, allow_untracked=True - ), - "config": { - "n_items": args.n_items, - "budget": args.budget, - "n_train_instances": args.n_train, - "epochs": args.epochs, - "n_seeds": args.seeds, - "n_features": n_features, - "feature_names": avail, - "lgd": LGD, - }, - "per_period": per_period_json, - "stability_summary": { - "coverage_always_above_target": all(c >= 0.90 for c in coverages_90), - "coverage_range": [round(min(coverages_90), 4), round(max(coverages_90), 4)], - "spo_improvement_range_pct": ( - [round(min(spo_improvements), 2), round(max(spo_improvements), 2)] - if spo_improvements - else None - ), - }, - "train_time_seconds": round(total_time, 1), - } + summary = _summary_payload( + run_tag=run_tag, + args=args, + n_features=n_features, + feature_names=avail, + rows=rows, + per_period_regrets=per_period_regrets, + total_time=total_time, + ) json_path = DATA_DIR / "crpto_vs_spo_stability.json" with open(json_path, "w") as f: diff --git a/scripts/run_fairness_audit.py b/scripts/run_fairness_audit.py index 6ed6206..03c9b2f 100644 --- a/scripts/run_fairness_audit.py +++ b/scripts/run_fairness_audit.py @@ -16,6 +16,7 @@ import argparse import json from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -42,6 +43,11 @@ SHAP_SAMPLE_SIZE = 10_000 +def _as_float(value: Any) -> float: + """Convert scalar config/data values to float with a narrow error surface.""" + return float(value) + + def _load_config(config_path: str) -> dict: """Load fairness policy YAML config.""" with open(config_path) as f: @@ -140,13 +146,14 @@ def _select_threshold_from_frontier( rows: list[dict[str, float | int]] = [] for threshold, grp in frontier.groupby("threshold", observed=True): + threshold_value = _as_float(threshold) rows.append( { - "threshold": float(threshold), + "threshold": threshold_value, "n_passed": int(grp["passed_all"].sum()), "worst_eo_gap": float(grp["eo_gap"].max()), "approval_rate": float( - (np.asarray(y_pred_proba_eval, dtype=float) >= float(threshold)).mean() + (np.asarray(y_pred_proba_eval, dtype=float) >= threshold_value).mean() ), } ) @@ -198,6 +205,211 @@ def _apply_decision_policy( return (y_pred_proba >= thresholds).astype(float) +def _model_feature_names(model: Any) -> list[str]: + return list(getattr(model, "feature_names_", None) or []) + + +def _available_features(feature_names: list[str], data: pd.DataFrame) -> list[str]: + return [feature for feature in feature_names if feature in data.columns] + + +def _sample_shap_frame( + frame: pd.DataFrame, + *, + shap_sample_size: int, + random_state: int, +) -> tuple[pd.DataFrame, np.ndarray]: + rng = np.random.default_rng(random_state) + n_rows = len(frame) + if n_rows > shap_sample_size: + sample_idx = np.sort(rng.choice(n_rows, size=shap_sample_size, replace=False)) + else: + sample_idx = np.arange(n_rows) + return frame.iloc[sample_idx].reset_index(drop=True), sample_idx + + +def _cat_features_from_model(model: Any, frame: pd.DataFrame) -> list[str]: + try: + feature_names = _model_feature_names(model) + cat_idx = model.get_cat_feature_indices() + return [ + feature_names[idx] + for idx in cat_idx + if idx < len(feature_names) and feature_names[idx] in frame.columns + ] + except Exception: + return [] + + +def _text_categorical_columns(frame: pd.DataFrame, known_cats: set[str]) -> list[str]: + detected: list[str] = [] + for col in frame.columns: + if col in known_cats or pd.api.types.is_numeric_dtype(frame[col]): + continue + probe = frame[col].dropna().head(5) + if probe.empty: + continue + try: + pd.to_numeric(probe, errors="raise") + except (ValueError, TypeError): + detected.append(str(col)) + known_cats.add(str(col)) + return detected + + +def _catboost_cat_feature_names(model: Any, frame: pd.DataFrame) -> list[str]: + cat_names = _cat_features_from_model(model, frame) + cat_set = set(cat_names) + cat_names.extend(_text_categorical_columns(frame, cat_set)) + return cat_names + + +def _prepare_catboost_shap_frame(frame: pd.DataFrame, cat_feature_names: list[str]) -> pd.DataFrame: + out = frame.copy() + cat_set = set(cat_feature_names) + for col in list(out.columns): + if not out[col].isna().any(): + continue + if col in cat_set: + out[col] = out[col].astype(object).fillna("missing").astype(str) + elif pd.api.types.is_numeric_dtype(out[col]): + out[col] = out[col].fillna(0.0) + return out + + +def _catboost_shap_matrix( + model: Any, frame: pd.DataFrame, cat_feature_names: list[str] +) -> np.ndarray: + from catboost import Pool as CatPool + + pool = CatPool(frame, cat_features=cat_feature_names or None) + shap_raw = model.get_feature_importance(pool, type="ShapValues") + return np.abs(np.asarray(shap_raw[:, :-1], dtype=float)) + + +def _top_shap_features( + mean_abs_shap: np.ndarray, + feature_names: list[str], + *, + limit: int, +) -> list[dict[str, object]]: + top_idx = np.argsort(mean_abs_shap)[::-1][:limit] + return [ + {"feature": feature_names[idx], "mean_abs_shap": float(mean_abs_shap[idx])} + for idx in top_idx + ] + + +def _pairwise_shap_diffs( + group_shap: dict[str, np.ndarray], + feature_names: list[str], + *, + limit: int, +) -> list[dict[str, object]]: + pairwise_diffs: list[dict[str, object]] = [] + groups_with_shap = list(group_shap.keys()) + for i, group_a in enumerate(groups_with_shap): + for group_b in groups_with_shap[i + 1 :]: + diff = np.abs(group_shap[group_a] - group_shap[group_b]) + top_idx = np.argsort(diff)[::-1][:limit] + pairwise_diffs.append( + { + "group_a": group_a, + "group_b": group_b, + "top_driving_features": [ + { + "feature": feature_names[idx], + "shap_diff": float(diff[idx]), + } + for idx in top_idx + ], + } + ) + return pairwise_diffs + + +def _attribute_shap_result( + *, + attribute: str, + labels: np.ndarray, + sample_idx: np.ndarray, + shap_matrix: np.ndarray, + feature_names: list[str], + min_group_size: int = 10, +) -> dict[str, object]: + group_labels = pd.Series(labels).iloc[sample_idx].reset_index(drop=True).astype(str) + unique_groups = sorted(group_labels.unique()) + group_shap: dict[str, np.ndarray] = {} + group_top5: dict[str, list[dict[str, object]]] = {} + + for group in unique_groups: + mask = group_labels.eq(group).to_numpy() + if mask.sum() < min_group_size: + continue + mean_abs_shap = shap_matrix[mask].mean(axis=0) + group_shap[str(group)] = mean_abs_shap + group_top5[str(group)] = _top_shap_features(mean_abs_shap, feature_names, limit=5) + + groups_with_shap = list(group_shap.keys()) + return { + "attribute": attribute, + "groups_analyzed": groups_with_shap, + "top5_per_group": group_top5, + "pairwise_feature_diffs": _pairwise_shap_diffs( + group_shap, + feature_names, + limit=3, + ), + } + + +def _shap_attribute_results( + groups_dict: dict[str, np.ndarray], + *, + sample_idx: np.ndarray, + shap_matrix: np.ndarray, + feature_names: list[str], +) -> list[dict[str, object]]: + attribute_results: list[dict[str, object]] = [] + for attribute, labels in groups_dict.items(): + if "__x__" in attribute: + continue + result = _attribute_shap_result( + attribute=attribute, + labels=labels, + sample_idx=sample_idx, + shap_matrix=shap_matrix, + feature_names=feature_names, + ) + attribute_results.append(result) + groups_analyzed = result.get("groups_analyzed", []) + n_groups = len(groups_analyzed) if isinstance(groups_analyzed, list) else 0 + logger.info(f"SHAP per-group: {attribute} ({n_groups} groups)") + return attribute_results + + +def _shap_result_payload( + *, + model_path: Path, + sample_size: int, + n_features: int, + attribute_results: list[dict[str, object]], +) -> dict[str, object]: + return { + "schema_version": SCHEMA_VERSION, + "model_path": str(model_path), + "shap_sample_size": sample_size, + "n_features": n_features, + "attributes": attribute_results, + "interpretation": ( + "For each protected attribute, top-5 features by mean |SHAP| per group. " + "Pairwise diffs show which features drive SHAP disparities between groups. " + "Features like dti/loan_amnt are legitimate credit risk factors; " + "home_ownership may proxy for race in US ECOA context." + ), + } + + def _compute_shap_per_group( data: pd.DataFrame, groups_dict: dict[str, np.ndarray], @@ -236,158 +448,378 @@ def _compute_shap_per_group( try: model = CatBoostClassifier() model.load_model(str(model_path)) - feature_names: list[str] = list(model.feature_names_) + feature_names = _model_feature_names(model) except Exception as e: logger.warning(f"SHAP per-group analysis skipped — model load error: {e}") return None - available_features = [f for f in feature_names if f in data.columns] + available_features = _available_features(feature_names, data) if not available_features: logger.warning("SHAP per-group analysis skipped — no model features found in test data") return None - X_full = data[available_features].copy() - - rng = np.random.default_rng(random_state) - n = len(X_full) - if n > shap_sample_size: - sample_idx = np.sort(rng.choice(n, size=shap_sample_size, replace=False)) - X_sample = X_full.iloc[sample_idx].reset_index(drop=True) - else: - sample_idx = np.arange(n) - X_sample = X_full.reset_index(drop=True) + x_sample, sample_idx = _sample_shap_frame( + data[available_features].copy(), + shap_sample_size=shap_sample_size, + random_state=random_state, + ) logger.info( - f"Computing SHAP values on {len(X_sample):,} rows, {len(available_features)} features" + f"Computing SHAP values on {len(x_sample):,} rows, {len(available_features)} features" ) try: # Use CatBoost's native SHAP via Pool + get_feature_importance — avoids the shap # library's cat/NaN handling issues entirely. model.get_cat_feature_indices() works # on .cbm-loaded models without needing the sklearn feature_names_ attribute. - import pandas as _pd - from catboost import Pool as _CatPool - - # Identify cat feature column names. - # Primary: use model metadata (works on .cbm loaded models when feature_names_ is set). - # Fallback: detect from data — any column whose non-null values cannot be cast to float - # is genuinely categorical (e.g. "very_high__E" WOE bin labels, grade strings). - _cat_feat_names: list[str] = [] - try: - _fn = list(getattr(model, "feature_names_", None) or []) - if _fn: - _cat_idx = model.get_cat_feature_indices() - _cat_feat_names = [ - _fn[i] for i in _cat_idx if i < len(_fn) and _fn[i] in X_sample.columns - ] - except Exception: - pass - - # Always supplement with content-based detection: any column whose non-null values - # cannot be cast to float is categorical, regardless of model metadata. - # This catches columns that have string values but are not in model.get_cat_feature_indices() - # (e.g. WOE bin labels stored as strings in the parquet instead of numeric WOE scores). - _cat_set = set(_cat_feat_names) - for _col in X_sample.columns: - if _col in _cat_set or _pd.api.types.is_numeric_dtype(X_sample[_col]): - continue - _probe = X_sample[_col].dropna().head(5) - if _probe.empty: - continue - try: - _pd.to_numeric(_probe, errors="raise") - except (ValueError, TypeError): - _cat_feat_names.append(_col) - _cat_set.add(_col) - - # Fill NaN: cat features → "missing", numeric features → 0.0 - # Only touch NaN cells; never alter non-NaN values. - X_sample = X_sample.copy() - _cat_set = set(_cat_feat_names) - for _col in list(X_sample.columns): - if not X_sample[_col].isna().any(): - continue - if _col in _cat_set: - X_sample[_col] = X_sample[_col].astype(object).fillna("missing").astype(str) - elif _pd.api.types.is_numeric_dtype(X_sample[_col]): - X_sample[_col] = X_sample[_col].fillna(0.0) - - pool = _CatPool(X_sample, cat_features=_cat_feat_names or None) - shap_raw = model.get_feature_importance(pool, type="ShapValues") - # get_feature_importance returns (n_samples, n_features + 1); last col is bias - shap_matrix = np.abs(np.asarray(shap_raw[:, :-1], dtype=float)) + cat_feature_names = _catboost_cat_feature_names(model, x_sample) + x_sample = _prepare_catboost_shap_frame(x_sample, cat_feature_names) + shap_matrix = _catboost_shap_matrix(model, x_sample, cat_feature_names) except Exception as e: logger.warning(f"SHAP per-group analysis skipped — SHAP computation error: {e}") return None - feature_names_sample = available_features - attribute_results: list[dict[str, object]] = [] + attribute_results = _shap_attribute_results( + groups_dict, + sample_idx=sample_idx, + shap_matrix=shap_matrix, + feature_names=available_features, + ) + return _shap_result_payload( + model_path=model_path, + sample_size=len(x_sample), + n_features=len(available_features), + attribute_results=attribute_results, + ) - for attribute, labels in groups_dict.items(): - if "__x__" in attribute: - continue - group_labels = pd.Series(labels).iloc[sample_idx].reset_index(drop=True).astype(str) - unique_groups = sorted(group_labels.unique()) - group_shap: dict[str, np.ndarray] = {} - group_top5: dict[str, list[dict[str, object]]] = {} - - for grp in unique_groups: - mask = group_labels == grp - if mask.sum() < 10: - continue - mean_abs_shap = shap_matrix[mask].mean(axis=0) - group_shap[grp] = mean_abs_shap - top5_idx = np.argsort(mean_abs_shap)[::-1][:5] - group_top5[grp] = [ - {"feature": feature_names_sample[i], "mean_abs_shap": float(mean_abs_shap[i])} - for i in top5_idx - ] - - pairwise_diffs: list[dict[str, object]] = [] - groups_with_shap = list(group_shap.keys()) - for i in range(len(groups_with_shap)): - for j in range(i + 1, len(groups_with_shap)): - g_a, g_b = groups_with_shap[i], groups_with_shap[j] - diff = np.abs(group_shap[g_a] - group_shap[g_b]) - top3_idx = np.argsort(diff)[::-1][:3] - pairwise_diffs.append( - { - "group_a": g_a, - "group_b": g_b, - "top_driving_features": [ - { - "feature": feature_names_sample[k], - "shap_diff": float(diff[k]), - } - for k in top3_idx - ], - } + +def _bootstrap_base_indices( + n_rows: int, + *, + bootstrap_max_rows: int, + rng: np.random.Generator, +) -> np.ndarray: + if bootstrap_max_rows > 0 and n_rows > bootstrap_max_rows: + return np.sort(rng.choice(n_rows, size=bootstrap_max_rows, replace=False)) + return np.arange(n_rows) + + +def _fairlearn_group_rows( + *, + attribute: str, + y_true: np.ndarray, + y_pred: np.ndarray, + sensitive: pd.Series, +) -> list[dict[str, object]]: + metric_frame = MetricFrame( + metrics={"selection_rate": selection_rate, "accuracy": accuracy_score}, + y_true=y_true, + y_pred=y_pred, + sensitive_features=sensitive, + ) + by_group = metric_frame.by_group.reset_index() + by_group.columns = ["group", *[str(col) for col in by_group.columns[1:]]] + rows = by_group.to_dict(orient="records") + for row in rows: + row["attribute"] = attribute + return rows + + +def _bootstrap_fairlearn_gaps( + *, + boot_sensitive_base: pd.Series, + boot_true_base: np.ndarray, + boot_pred_base: np.ndarray, + rng: np.random.Generator, + n_boot: int, +) -> tuple[list[float], list[float]]: + dpd_boot: list[float] = [] + eo_boot: list[float] = [] + for _ in range(max(n_boot, 0)): + idx = rng.integers(0, len(boot_true_base), len(boot_true_base)) + boot_sensitive = boot_sensitive_base.iloc[idx] + boot_true = boot_true_base[idx] + boot_pred = boot_pred_base[idx] + dpd_boot.append( + float( + demographic_parity_difference( + y_true=boot_true, + y_pred=boot_pred, + sensitive_features=boot_sensitive, + ) + ) + ) + eo_boot.append( + float( + equalized_odds_difference( + y_true=boot_true, + y_pred=boot_pred, + sensitive_features=boot_sensitive, ) + ) + ) + return dpd_boot, eo_boot - attribute_results.append( - { - "attribute": attribute, - "groups_analyzed": groups_with_shap, - "top5_per_group": group_top5, - "pairwise_feature_diffs": pairwise_diffs, - } + +def _fairlearn_summary_row( + *, + attribute: str, + y_true: np.ndarray, + y_pred: np.ndarray, + sensitive: pd.Series, + bootstrap_idx: np.ndarray, + rng: np.random.Generator, + n_boot: int, +) -> dict[str, object]: + dpd = float( + demographic_parity_difference( + y_true=y_true, + y_pred=y_pred, + sensitive_features=sensitive, + ) + ) + eo = float( + equalized_odds_difference( + y_true=y_true, + y_pred=y_pred, + sensitive_features=sensitive, ) - logger.info(f"SHAP per-group: {attribute} ({len(groups_with_shap)} groups)") + ) + dpd_boot, eo_boot = _bootstrap_fairlearn_gaps( + boot_sensitive_base=sensitive.iloc[bootstrap_idx].reset_index(drop=True), + boot_true_base=y_true[bootstrap_idx], + boot_pred_base=y_pred[bootstrap_idx], + rng=rng, + n_boot=n_boot, + ) + return { + "attribute": attribute, + "demographic_parity_difference": dpd, + "equalized_odds_difference": eo, + "dpd_ci_low": float(np.quantile(dpd_boot, 0.025)) if dpd_boot else None, + "dpd_ci_high": float(np.quantile(dpd_boot, 0.975)) if dpd_boot else None, + "eo_ci_low": float(np.quantile(eo_boot, 0.025)) if eo_boot else None, + "eo_ci_high": float(np.quantile(eo_boot, 0.975)) if eo_boot else None, + } + + +def _fairlearn_sidecar_rows( + *, + groups_all: dict[str, np.ndarray], + y_true: np.ndarray, + y_pred: np.ndarray, + bootstrap_idx: np.ndarray, + rng: np.random.Generator, + n_boot: int, +) -> tuple[list[dict[str, object]], list[dict[str, object]]]: + group_rows: list[dict[str, object]] = [] + summary_rows: list[dict[str, object]] = [] + for attribute, labels in groups_all.items(): + sensitive = pd.Series(labels).astype(str).reset_index(drop=True) + group_rows.extend( + _fairlearn_group_rows( + attribute=attribute, + y_true=y_true, + y_pred=y_pred, + sensitive=sensitive, + ) + ) + summary_rows.append( + _fairlearn_summary_row( + attribute=attribute, + y_true=y_true, + y_pred=y_pred, + sensitive=sensitive, + bootstrap_idx=bootstrap_idx, + rng=rng, + n_boot=n_boot, + ) + ) + return group_rows, summary_rows + + +def _write_fairlearn_sidecar( + *, + sidecar_cfg: dict, + groups_all: dict[str, np.ndarray], + y_true_eval: np.ndarray, + y_pred_binary: np.ndarray, + status_path: Path, + primary_threshold: float, + outcome_mode: str, + resolved_run_tag: str, +) -> None: + rng = np.random.default_rng(int(sidecar_cfg.get("bootstrap_random_state", 42))) + n_boot = int(sidecar_cfg.get("bootstrap_samples", 200)) + bootstrap_max_rows = int(sidecar_cfg.get("bootstrap_max_rows", 50_000)) + y_true_arr = np.asarray(y_true_eval, dtype=float) + y_pred_arr = np.asarray(y_pred_binary, dtype=float) + bootstrap_idx = _bootstrap_base_indices( + len(y_true_arr), + bootstrap_max_rows=bootstrap_max_rows, + rng=rng, + ) + group_rows, summary_rows = _fairlearn_sidecar_rows( + groups_all=groups_all, + y_true=y_true_arr, + y_pred=y_pred_arr, + bootstrap_idx=bootstrap_idx, + rng=rng, + n_boot=n_boot, + ) + + sidecar_path = Path(sidecar_cfg.get("status_json", "models/fairlearn_fairness_status.json")) + group_metrics_path = Path( + sidecar_cfg.get("group_metrics_parquet", "data/processed/fairlearn_group_metrics.parquet") + ) + group_metrics_path.parent.mkdir(parents=True, exist_ok=True) + sidecar_path.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame(group_rows).to_parquet(group_metrics_path, index=False) + sidecar_payload = { + "primary_status_path": str(status_path), + "group_metrics_path": str(group_metrics_path), + "n_attributes": len(summary_rows), + "attributes": summary_rows, + "bootstrap_samples": n_boot, + "bootstrap_rows_used": len(bootstrap_idx), + "bootstrap_max_rows": bootstrap_max_rows, + "prediction_threshold": float(primary_threshold), + "outcome_mode": outcome_mode, + **build_artifact_metadata( + schema_version=f"{SCHEMA_VERSION}-fairlearn", + run_tag=resolved_run_tag, + require_explicit=True, + ), + } + sidecar_path.write_text(json.dumps(sidecar_payload, indent=2, default=str), encoding="utf-8") + logger.info(f"Saved fairlearn sidecar status: {sidecar_path}") + + +def _with_attribute_type(frame: pd.DataFrame) -> pd.DataFrame: + if frame.empty: + return frame + out = frame.copy() + out["attribute_type"] = np.where( + out["attribute"].astype(str).str.contains("__x__"), + "intersectional", + "base", + ) + return out + + +def _primary_frontier(frontier: pd.DataFrame, primary_threshold: float) -> pd.DataFrame: + if frontier.empty: + return pd.DataFrame() + return frontier.loc[np.isclose(frontier["threshold"].astype(float), primary_threshold)] + + +def _worst_primary_attribute(primary_frontier: pd.DataFrame) -> str: + if primary_frontier.empty: + return "" + return str( + primary_frontier.sort_values( + by=["passed_all", "eo_gap", "dpd", "dir"], + ascending=[True, False, False, True], + ).iloc[0]["attribute"] + ) + +def _decision_override_count(decision_policy: Any) -> int: + overrides = decision_policy.get("overrides", []) if isinstance(decision_policy, dict) else [] + return len(overrides) if isinstance(overrides, list) else 0 + + +def _decision_global_threshold(decision_policy: Any, primary_threshold: float) -> float: + if isinstance(decision_policy, dict): + return _as_float(decision_policy.get("global_threshold", primary_threshold)) + return float(primary_threshold) + + +def _fairness_status_payload( + *, + report: pd.DataFrame, + frontier: pd.DataFrame, + frontier_path: Path, + frontier_thresholds: list[float], + primary_threshold: float, + threshold_source: str, + outcome_mode: str, + policy: dict, + decision_policy: Any, + decision_policy_path: Path, + config_path: str, + resolved_run_tag: str, +) -> dict[str, object]: + primary_frontier = _primary_frontier(frontier, primary_threshold) return { - "schema_version": SCHEMA_VERSION, - "model_path": str(model_path), - "shap_sample_size": len(X_sample), - "n_features": len(available_features), - "attributes": attribute_results, - "interpretation": ( - "For each protected attribute, top-5 features by mean |SHAP| per group. " - "Pairwise diffs show which features drive SHAP disparities between groups. " - "Features like dti/loan_amnt are legitimate credit risk factors; " - "home_ownership may proxy for race in US ECOA context." + "overall_pass": bool(report["passed_all"].all()), + "n_attributes": len(report), + "n_base_attributes": int( + (report.get("attribute_type", pd.Series(dtype=str)) == "base").sum() + ), + "n_intersectional_attributes": int( + (report.get("attribute_type", pd.Series(dtype=str)) == "intersectional").sum() + ), + "n_passed": int(report["passed_all"].sum()), + "attributes": report.to_dict(orient="records"), + "prediction_threshold": float(primary_threshold), + "primary_threshold": float(primary_threshold), + "prediction_threshold_source": threshold_source, + "outcome_mode": outcome_mode, + "thresholds": { + "dpd": policy["dpd_threshold"], + "eo_gap": policy["eo_gap_threshold"], + "dir": policy["dir_threshold"], + }, + "threshold_frontier": { + "path": str(frontier_path), + "thresholds": frontier_thresholds, + "worst_primary_attribute": _worst_primary_attribute(primary_frontier), + "selected_threshold": float(primary_threshold), + "all_primary_pass": bool( + primary_frontier.get("passed_all", pd.Series(dtype=bool)).all() + ) + if not primary_frontier.empty + else True, + }, + "decision_policy": { + "path": str(decision_policy_path), + "global_threshold": float(primary_threshold), + "n_overrides": _decision_override_count(decision_policy), + }, + "policy_config": str(config_path), + **build_artifact_metadata( + schema_version=SCHEMA_VERSION, + run_tag=resolved_run_tag, + require_explicit=True, ), } +def _write_json_payload(path: Path, payload: dict[str, object], *, label: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + logger.info(f"Saved {label}: {path}") + + +def _write_shap_status( + *, + shap_result: dict[str, object] | None, + resolved_run_tag: str, + primary_threshold: float, + outcome_mode: str, + path: Path = SHAP_STATUS_PATH, +) -> None: + if shap_result is None: + return + shap_result["generated_at_utc"] = str( + __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat() + ) + shap_result["run_tag"] = resolved_run_tag + shap_result["prediction_threshold"] = float(primary_threshold) + shap_result["outcome_mode"] = outcome_mode + _write_json_payload(path, shap_result, label="SHAP per-group fairness analysis") + + def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None = None) -> None: """Run the fairness audit pipeline.""" cfg = _load_config(config_path) @@ -467,12 +899,7 @@ def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None eo_gap_threshold=policy["eo_gap_threshold"], dir_threshold=policy["dir_threshold"], ) - if not frontier.empty: - frontier["attribute_type"] = np.where( - frontier["attribute"].astype(str).str.contains("__x__"), - "intersectional", - "base", - ) + frontier = _with_attribute_type(frontier) frontier_path = Path( output.get("frontier_parquet", "data/processed/fairness_threshold_frontier.parquet") ) @@ -537,223 +964,60 @@ def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None eo_gap_threshold=policy["eo_gap_threshold"], dir_threshold=policy["dir_threshold"], ) - if not report.empty: - report["attribute_type"] = np.where( - report["attribute"].astype(str).str.contains("__x__"), - "intersectional", - "base", - ) + report = _with_attribute_type(report) audit_path = Path(output["audit_parquet"]) audit_path.parent.mkdir(parents=True, exist_ok=True) report.to_parquet(audit_path, index=False) logger.info(f"Saved fairness audit: {audit_path}") - # Build and save status JSON - overall_pass = bool(report["passed_all"].all()) - primary_frontier = ( - frontier.loc[np.isclose(frontier["threshold"].astype(float), primary_threshold)] - if not frontier.empty - else pd.DataFrame() + status = _fairness_status_payload( + report=report, + frontier=frontier, + frontier_path=frontier_path, + frontier_thresholds=frontier_thresholds, + primary_threshold=float(primary_threshold), + threshold_source=threshold_source, + outcome_mode=outcome_mode, + policy=policy, + decision_policy=decision_policy, + decision_policy_path=decision_policy_path, + config_path=config_path, + resolved_run_tag=resolved_run_tag, ) - worst_primary_attribute = "" - if not primary_frontier.empty: - worst_primary_attribute = str( - primary_frontier.sort_values( - by=["passed_all", "eo_gap", "dpd", "dir"], - ascending=[True, False, False, True], - ).iloc[0]["attribute"] - ) - status = { - "overall_pass": overall_pass, - "n_attributes": len(report), - "n_base_attributes": int( - (report.get("attribute_type", pd.Series(dtype=str)) == "base").sum() - ), - "n_intersectional_attributes": int( - (report.get("attribute_type", pd.Series(dtype=str)) == "intersectional").sum() - ), - "n_passed": int(report["passed_all"].sum()), - "attributes": report.to_dict(orient="records"), - "prediction_threshold": float(primary_threshold), - "primary_threshold": float(primary_threshold), - "prediction_threshold_source": threshold_source, - "outcome_mode": outcome_mode, - "thresholds": { - "dpd": policy["dpd_threshold"], - "eo_gap": policy["eo_gap_threshold"], - "dir": policy["dir_threshold"], - }, - "threshold_frontier": { - "path": str(frontier_path), - "thresholds": frontier_thresholds, - "worst_primary_attribute": worst_primary_attribute, - "selected_threshold": float(primary_threshold), - "all_primary_pass": bool( - primary_frontier.get("passed_all", pd.Series(dtype=bool)).all() - ) - if not primary_frontier.empty - else True, - }, - "decision_policy": { - "path": str(decision_policy_path), - "global_threshold": float(primary_threshold), - "n_overrides": len(decision_policy.get("overrides", [])) - if isinstance(decision_policy, dict) - else 0, - }, - "policy_config": str(config_path), - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - } status_path = Path(output["status_json"]) - status_path.parent.mkdir(parents=True, exist_ok=True) - with open(status_path, "w", encoding="utf-8") as f: - json.dump(status, f, indent=2, default=str) - logger.info(f"Saved fairness status: {status_path}") + _write_json_payload(status_path, status, label="fairness status") sidecar_cfg = cfg.get("fairlearn_sidecar", {}) or {} if bool(sidecar_cfg.get("enabled", True)): - group_rows: list[dict[str, object]] = [] - summary_rows: list[dict[str, object]] = [] - rng = np.random.default_rng(int(sidecar_cfg.get("bootstrap_random_state", 42))) - n_boot = int(sidecar_cfg.get("bootstrap_samples", 200)) - bootstrap_max_rows = int(sidecar_cfg.get("bootstrap_max_rows", 50_000)) - y_true_arr = np.asarray(y_true_eval, dtype=float) - y_pred_arr = np.asarray(y_pred_binary, dtype=float) - if bootstrap_max_rows > 0 and len(y_true_arr) > bootstrap_max_rows: - bootstrap_idx = np.sort( - rng.choice(len(y_true_arr), size=bootstrap_max_rows, replace=False) - ) - else: - bootstrap_idx = np.arange(len(y_true_arr)) - - for attribute, labels in groups_all.items(): - sensitive = pd.Series(labels).astype(str).reset_index(drop=True) - mf = MetricFrame( - metrics={"selection_rate": selection_rate, "accuracy": accuracy_score}, - y_true=y_true_arr, - y_pred=y_pred_arr, - sensitive_features=sensitive, - ) - by_group = mf.by_group.reset_index() - by_group.columns = ["group", *[str(col) for col in by_group.columns[1:]]] - for row in by_group.to_dict(orient="records"): - row["attribute"] = attribute - group_rows.append(row) - - dpd = float( - demographic_parity_difference( - y_true=y_true_arr, - y_pred=y_pred_arr, - sensitive_features=sensitive, - ) - ) - eo = float( - equalized_odds_difference( - y_true=y_true_arr, - y_pred=y_pred_arr, - sensitive_features=sensitive, - ) - ) - boot_sensitive_base = sensitive.iloc[bootstrap_idx].reset_index(drop=True) - boot_true_base = y_true_arr[bootstrap_idx] - boot_pred_base = y_pred_arr[bootstrap_idx] - dpd_boot: list[float] = [] - eo_boot: list[float] = [] - for _ in range(max(n_boot, 0)): - idx = rng.integers(0, len(boot_true_base), len(boot_true_base)) - boot_sensitive = boot_sensitive_base.iloc[idx] - boot_true = boot_true_base[idx] - boot_pred = boot_pred_base[idx] - dpd_boot.append( - float( - demographic_parity_difference( - y_true=boot_true, - y_pred=boot_pred, - sensitive_features=boot_sensitive, - ) - ) - ) - eo_boot.append( - float( - equalized_odds_difference( - y_true=boot_true, - y_pred=boot_pred, - sensitive_features=boot_sensitive, - ) - ) - ) - summary_rows.append( - { - "attribute": attribute, - "demographic_parity_difference": dpd, - "equalized_odds_difference": eo, - "dpd_ci_low": float(np.quantile(dpd_boot, 0.025)) if dpd_boot else None, - "dpd_ci_high": float(np.quantile(dpd_boot, 0.975)) if dpd_boot else None, - "eo_ci_low": float(np.quantile(eo_boot, 0.025)) if eo_boot else None, - "eo_ci_high": float(np.quantile(eo_boot, 0.975)) if eo_boot else None, - } - ) - - sidecar_path = Path(sidecar_cfg.get("status_json", "models/fairlearn_fairness_status.json")) - group_metrics_path = Path( - sidecar_cfg.get( - "group_metrics_parquet", "data/processed/fairlearn_group_metrics.parquet" - ) - ) - group_metrics_path.parent.mkdir(parents=True, exist_ok=True) - sidecar_path.parent.mkdir(parents=True, exist_ok=True) - pd.DataFrame(group_rows).to_parquet(group_metrics_path, index=False) - sidecar_payload = { - "primary_status_path": str(status_path), - "group_metrics_path": str(group_metrics_path), - "n_attributes": len(summary_rows), - "attributes": summary_rows, - "bootstrap_samples": n_boot, - "bootstrap_rows_used": len(bootstrap_idx), - "bootstrap_max_rows": bootstrap_max_rows, - "prediction_threshold": float(primary_threshold), - "outcome_mode": outcome_mode, - **build_artifact_metadata( - schema_version=f"{SCHEMA_VERSION}-fairlearn", - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - sidecar_path.write_text( - json.dumps(sidecar_payload, indent=2, default=str), encoding="utf-8" + _write_fairlearn_sidecar( + sidecar_cfg=sidecar_cfg, + groups_all=groups_all, + y_true_eval=y_true_eval, + y_pred_binary=y_pred_binary, + status_path=status_path, + primary_threshold=float(primary_threshold), + outcome_mode=outcome_mode, + resolved_run_tag=resolved_run_tag, ) - logger.info(f"Saved fairlearn sidecar status: {sidecar_path}") - shap_result = _compute_shap_per_group( - data=data.iloc[:n], - groups_dict=groups_dict, + _write_shap_status( + shap_result=_compute_shap_per_group( + data=data.iloc[:n], + groups_dict=groups_dict, + ), + resolved_run_tag=resolved_run_tag, + primary_threshold=float(primary_threshold), + outcome_mode=outcome_mode, ) - if shap_result is not None: - shap_result["generated_at_utc"] = str( - __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat() - ) - shap_result["run_tag"] = resolved_run_tag - shap_result["prediction_threshold"] = float(primary_threshold) - shap_result["outcome_mode"] = outcome_mode - SHAP_STATUS_PATH.parent.mkdir(parents=True, exist_ok=True) - SHAP_STATUS_PATH.write_text( - json.dumps(shap_result, indent=2, default=str), encoding="utf-8" - ) - logger.info(f"Saved SHAP per-group fairness analysis: {SHAP_STATUS_PATH}") write_threshold_semantics( fairness_primary_threshold=float(primary_threshold), - decision_policy_global_threshold=float( - decision_policy.get("global_threshold", primary_threshold) - ) - if isinstance(decision_policy, dict) - else float(primary_threshold), + decision_policy_global_threshold=_decision_global_threshold( + decision_policy, + float(primary_threshold), + ), source_artifacts={ "fairness_status": str(status_path), "fairness_decision_policy": str(decision_policy_path), @@ -767,6 +1031,7 @@ def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None path=output.get("threshold_semantics_json", "models/threshold_semantics.json"), ) + overall_pass = bool(status["overall_pass"]) pass_label = "PASS" if overall_pass else "FAIL" logger.info( f"Fairness audit: {pass_label} ({status['n_passed']}/{status['n_attributes']} attributes)" diff --git a/scripts/run_spo_comparison.py b/scripts/run_spo_comparison.py index 98a83da..8f46af7 100644 --- a/scripts/run_spo_comparison.py +++ b/scripts/run_spo_comparison.py @@ -13,6 +13,7 @@ from __future__ import annotations import argparse +import importlib import json import sys from datetime import UTC, datetime @@ -117,8 +118,8 @@ def _try_spo_comparison( ) -> dict[str, object] | None: """Attempt SPO+ comparison if pyepo and torch are available.""" try: - import torch # noqa: F401 - from pyepo.func import SPOPlus # noqa: F401 + importlib.import_module("torch") + importlib.import_module("pyepo.func") logger.info("PyEPO and torch available. SPO+ comparison enabled.") except ImportError: diff --git a/scripts/run_spo_real.py b/scripts/run_spo_real.py index d451630..e4adfef 100644 --- a/scripts/run_spo_real.py +++ b/scripts/run_spo_real.py @@ -18,6 +18,7 @@ from __future__ import annotations import argparse +import importlib import json import pickle import time @@ -26,12 +27,8 @@ import numpy as np import pandas as pd -import pyepo -import torch -import torch.nn as nn from loguru import logger from ortools.linear_solver import pywraplp -from pyepo.model.opt import optModel from scipy import stats from src.models.conformal import apply_probability_calibrator @@ -41,6 +38,23 @@ LGD = 0.40 RANDOM_SEED = 42 + +def _require_optional_module(module_name: str) -> Any: + """Import an optional SPO dependency with an explicit experiment-level error.""" + try: + return importlib.import_module(module_name) + except ImportError as exc: + raise RuntimeError( + "SPO+ is an optional experiment. Install the `spo` extras before " + f"running this script; missing module: {module_name}." + ) from exc + + +pyepo: Any = _require_optional_module("pyepo") +torch: Any = _require_optional_module("torch") +nn: Any = _require_optional_module("torch.nn") +OptModelBase: Any = _require_optional_module("pyepo.model.opt").optModel + NUMERIC_FEATURES = [ "loan_amnt", "int_rate", @@ -63,7 +77,7 @@ # ── 1. Portfolio LP optModel ──────────────────────────────────────────────── -class CreditPortfolioLP(optModel): +class CreditPortfolioLP(OptModelBase): """Portfolio selection LP: select exactly `budget` of `n_items` loans to minimize expected loss c^T x = sum(x_i * (PD_i * LGD - r_i)). @@ -87,7 +101,7 @@ def _getModel(self) -> tuple: ct.SetCoefficient(x[i], 1.0) return solver, x - def setObj(self, c: np.ndarray | torch.Tensor) -> None: + def setObj(self, c: np.ndarray | Any) -> None: if isinstance(c, torch.Tensor): c = c.detach().cpu().numpy() c = np.asarray(c, dtype=float) @@ -139,7 +153,7 @@ def __init__(self, n_features: int, n_items: int) -> None: nn.Linear(32, 1), ) - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: Any) -> Any: B = x.shape[0] # (B, n_items * n_features) → (B * n_items, n_features) x_per_loan = x.view(B * self.n_items, self.n_features) @@ -345,9 +359,9 @@ def _train_spo( X_inst_train: (n_inst, n_items, n_features) — instance features. c_inst_train: (n_inst, n_items) — true costs per instance. """ - from pyepo.data.dataset import optDataset - from pyepo.func import SPOPlus - from torch.utils.data import DataLoader + optDataset = _require_optional_module("pyepo.data.dataset").optDataset + SPOPlus = _require_optional_module("pyepo.func").SPOPlus + DataLoader = _require_optional_module("torch.utils.data").DataLoader n_input = n_items * n_features X_flat = X_inst_train.reshape(len(X_inst_train), n_input) diff --git a/scripts/run_ty_advisory.py b/scripts/run_ty_advisory.py new file mode 100644 index 0000000..ab6736c --- /dev/null +++ b/scripts/run_ty_advisory.py @@ -0,0 +1,138 @@ +"""Run pinned ty as an advisory or blocking checker for CRPTO.""" + +from __future__ import annotations + +import argparse +import re +import shutil +import subprocess +from collections.abc import Sequence +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_OUTPUT = ROOT / "reports" / "ci" / "ty-advisory.txt" +TY_REQUIREMENT = "ty==0.0.57" + +SOURCE_ROOTS = ("src", "scripts") +ACTIVE_EXCLUDED_FILES = { + "scripts/generate_conformal_intervals.py", + "scripts/run_cqr_comparison.py", + "scripts/run_crpto_vs_spo_stability.py", + "scripts/run_spo_comparison.py", + "scripts/run_spo_real.py", + "scripts/train_pd_model.py", + "src/optimization/cuopt_adapter.py", +} +ACTIVE_EXPERIMENT_FILES = { + "scripts/experiments/ijds_policy_support.py", + "scripts/experiments/run_ijds_calibration_selected_policy_challenger.py", + "scripts/experiments/run_ijds_exact_alpha_grid_challenger.py", +} +SUMMARY_RE = re.compile(r"^Found \d+ diagnostics", flags=re.MULTILINE) + + +def _relative_posix(path: Path) -> str: + return path.relative_to(ROOT).as_posix() + + +def iter_python_files(*, scope: str) -> list[str]: + """Return the Python files ty should check for a given advisory scope.""" + files: list[str] = [] + for root_name in SOURCE_ROOTS: + for path in sorted((ROOT / root_name).rglob("*.py")): + rel = _relative_posix(path) + parts = rel.split("/") + if scope == "active": + if parts[:2] == ["scripts", "archive"]: + continue + if parts[:2] == ["scripts", "experiments"] and rel not in (ACTIVE_EXPERIMENT_FILES): + continue + if parts[:2] == ["scripts", "search"] and path.name.startswith("run_"): + continue + if rel in ACTIVE_EXCLUDED_FILES: + continue + files.append(rel) + return files + + +def build_ty_command(*, uvx: str, files: Sequence[str], fail_on_diagnostics: bool) -> list[str]: + """Build the pinned ty command for advisory or blocking use.""" + command = [ + uvx, + "--from", + TY_REQUIREMENT, + "ty", + "check", + "--python", + ".venv", + "--output-format", + "concise", + "--no-progress", + ] + if not fail_on_diagnostics: + command.append("--exit-zero") + return [*command, *files] + + +def run_ty(scope: str, output: Path, *, fail_on_diagnostics: bool = False) -> int: + """Run pinned ty, persist its report, and optionally enforce diagnostics.""" + uvx = shutil.which("uvx") + if uvx is None: + raise RuntimeError("uvx is required to run the ty advisory check.") + + files = iter_python_files(scope=scope) + command = build_ty_command( + uvx=uvx, + files=files, + fail_on_diagnostics=fail_on_diagnostics, + ) + result = subprocess.run( + command, + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + output.parent.mkdir(parents=True, exist_ok=True) + report = ( + f"# ty advisory report\n" + f"requirement: {TY_REQUIREMENT}\n" + f"scope: {scope}\n" + f"blocking: {str(fail_on_diagnostics).lower()}\n" + f"files_checked: {len(files)}\n" + f"return_code: {result.returncode}\n" + f"\n" + f"{result.stdout}{result.stderr}" + ) + output.write_text(report, encoding="utf-8") + summary = SUMMARY_RE.search(report) + if summary: + print(summary.group(0)) + elif result.returncode == 0: + print("ty advisory clean") + else: + print(f"ty failed with return code {result.returncode}") + print(f"Full report: {output.relative_to(ROOT)}") + return result.returncode if fail_on_diagnostics else 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--scope", choices=["active", "full"], default="active") + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument( + "--fail-on-diagnostics", + action="store_true", + help="Return ty's nonzero status when diagnostics are present.", + ) + args = parser.parse_args() + output = args.output if args.output.is_absolute() else ROOT / args.output + return run_ty( + scope=args.scope, + output=output, + fail_on_diagnostics=args.fail_on_diagnostics, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/search/build_pool93_body_allocation_audit.py b/scripts/search/build_pool93_body_allocation_audit.py index 526dc75..2d4d365 100644 --- a/scripts/search/build_pool93_body_allocation_audit.py +++ b/scripts/search/build_pool93_body_allocation_audit.py @@ -29,7 +29,14 @@ _compute_intervals_at_alpha, _load_aligned_dataset, ) -from src.optimization.portfolio_model import optimize_portfolio_allocation # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + compute_funded_certificate_metrics, +) +from src.optimization.portfolio_model import ( # noqa: E402 + optimize_portfolio_allocation, + solution_allocation_vector, +) +from src.utils.script_helpers import resolve_repo_artifact_path # noqa: E402 DEFAULT_CONSOLIDATED_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" DEFAULT_BODY_ROLE = "body/default balanced return-bound point" @@ -86,11 +93,17 @@ def _policy_from_row(row: dict[str, Any]) -> dict[str, Any]: def _grade_bucket(series: pd.Series) -> pd.Series: grade = series.fillna("unknown").astype(str).str.upper().str[:1] - return np.select( - [grade.isin(["A", "B"]), grade.eq("C"), grade.eq("D"), grade.isin(["E", "F", "G"])], + buckets = np.select( + [ + grade.isin(["A", "B"]), + grade.eq("C"), + grade.eq("D"), + grade.isin(["E", "F", "G"]), + ], ["A-B", "C", "D", "E-G"], default="unknown", ) + return pd.Series(buckets, index=series.index, name="grade_bucket") def _format_tex_table(summary: pd.DataFrame) -> str: @@ -100,9 +113,11 @@ def _format_tex_table(summary: pd.DataFrame) -> str: "Grade bucket & Funded rows & Exposure share & Default rate & $V$ contribution & Mean $u_i(0.01)$ \\\\", "\\midrule", ] - for row in summary.itertuples(index=False): + for row in summary.to_dict("records"): lines.append( - f"{row.grade_bucket} & {row.funded_rows:,.0f} & {row.exposure_share:.2%} & {row.default_rate:.2%} & {row.v_contribution:.5f} & {row.mean_pd_high_alpha01:.5f} \\\\" + f"{row['grade_bucket']} & {row['funded_rows']:,.0f} & " + f"{row['exposure_share']:.2%} & {row['default_rate']:.2%} & " + f"{row['v_contribution']:.5f} & {row['mean_pd_high_alpha01']:.5f} \\\\" ) lines.extend(["\\bottomrule", "\\end{tabular}", ""]) return "\n".join(lines) @@ -124,7 +139,9 @@ def build_audit( policy["solver_backend"] = solver_backend manifest = json.loads(_manifest_path(str(row["run_tag"])).read_text(encoding="utf-8")) - conformal_intervals_path = str(manifest["conformal_intervals_path"]) + conformal_intervals_path = str( + resolve_repo_artifact_path(manifest["conformal_intervals_path"], root=ROOT) + ) aligned = _load_aligned_dataset( conformal_intervals_path=conformal_intervals_path, max_candidates=int(manifest.get("max_candidates", 0) or 0), @@ -172,19 +189,23 @@ def build_audit( threads=max(1, int(threads)), solver_backend=str(policy["solver_backend"]), ) - if "allocation_vector" in solution: - alloc = np.asarray(solution["allocation_vector"], dtype=float) - else: - alloc = np.array( - [float(solution["allocation"].get(i, 0.0)) for i in range(len(aligned))], - dtype=float, - ) + alloc = solution_allocation_vector(solution, len(aligned)) exposure = alloc * loan_amounts total_allocated = float(exposure.sum()) weights = exposure / max(total_allocated, 1e-6) funded = alloc > 0.01 miscoverage = (y_true > pd_high).astype(float) + certificate = compute_funded_certificate_metrics( + weights, + outcomes=y_true, + pd_point=pd_point, + pd_high=pd_high, + pd_effective=effective_pd, + alpha=alpha, + risk_tolerance=float(policy["risk_tolerance"]), + pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), + ) realized_return = np.where( funded & (default_flag.astype(int) == 1), exposure * (-DEFAULT_LGD), @@ -255,23 +276,17 @@ def build_audit( "n_funded": int(funded.sum()), "total_allocated": round(total_allocated, 6), "realized_return": round(float(realized_return.sum()), 6), - "Gamma_CP": round(float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))), 6), - "V": round(float(np.sum(weights * miscoverage)), 6), - "weighted_pd_true": round(float(np.sum(weights * y_true)), 6), - "endpoint_budget_upper": round( - float(policy["risk_tolerance"]) - + (1.0 - float(policy["gamma"])) - * float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))), - 9, - ), - "markov_cap": round( - float(policy["risk_tolerance"]) - + (1.0 - float(policy["gamma"])) - * float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))) - + float(np.sqrt(alpha)), - 9, - ), - "empirical_coverage_funded": round(float(1.0 - miscoverage[funded].mean()), 6), + "Gamma_CP": round(certificate.gamma_cp, 6), + "Gamma_internalized": round(certificate.gamma_internalized, 6), + "Gamma_residual": round(certificate.gamma_residual, 6), + "V": round(certificate.weighted_miscoverage, 6), + "weighted_coverage_funded": round(certificate.weighted_coverage, 6), + "weighted_pd_true": round(certificate.weighted_outcome, 6), + "endpoint_budget": round(certificate.endpoint_budget, 9), + "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), + "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), + "markov_cap": round(certificate.markov_loss_cap, 9), + "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 6), }, "outputs": { "funded_rows": str(funded_path), diff --git a/scripts/search/build_pool93_ijds_consolidated_frontier.py b/scripts/search/build_pool93_ijds_consolidated_frontier.py index 83fc6d7..7b2d9ad 100644 --- a/scripts/search/build_pool93_ijds_consolidated_frontier.py +++ b/scripts/search/build_pool93_ijds_consolidated_frontier.py @@ -8,17 +8,23 @@ from pathlib import Path from typing import Any +import numpy as np import pandas as pd +from src.optimization.certificate_semantics import add_policy_aware_bound_columns + ROOT = Path(__file__).resolve().parents[2] -DEFAULT_OUTPUT_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated" +DEFAULT_OUTPUT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" DEFAULT_RUN_TAGS = [ "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", ] -DEFAULT_CAPS = [0.32, 0.33, 0.335, 0.34, 0.345, 0.35, 0.36, 0.45, 0.50] -DEFAULT_BODY_MARKOV_CAP = 0.35 +DEFAULT_CAPS = [0.30, 0.32, 0.345, 0.36, 0.45] +DEFAULT_BODY_MARKOV_THRESHOLD = 0.35 def _leaderboard_path(run_tag: str) -> Path: @@ -30,6 +36,15 @@ def _leaderboard_path(run_tag: str) -> Path: ) +def _bound_eval_path(run_tag: str) -> Path: + return ( + ROOT + / "data/processed/experiments/champion_reopen" + / run_tag + / "portfolio/pool93_ijds_local_refinement_bound_eval.parquet" + ) + + def _output_path(output_tag: str) -> Path: return ( ROOT @@ -46,6 +61,12 @@ def _short_run_label(run_tag: str) -> str: return "micro" if run_tag.endswith("__pool93__ijds-claim-micro-ext"): return "micro_ext" + if run_tag.endswith("__pool93__ijds-claim-bound-closure"): + return "bound_closure" + if run_tag.endswith("__pool93__ijds-claim-bound-floor-closure"): + return "bound_floor" + if run_tag.endswith("__pool93__ijds-claim-bound-terminal"): + return "bound_terminal" return run_tag @@ -67,8 +88,12 @@ def _row_payload(row: pd.Series, role: str) -> dict[str, Any]: "return": round(float(row["alpha01_realized_total_return"]), 6), "return_floor_surplus": round(float(row["return_floor_surplus"]), 6), "Gamma_CP": round(float(row["alpha01_gamma_cp"]), 6), + "Gamma_internalized": round(float(row["alpha01_gamma_internalized"]), 6), + "Gamma_residual": round(float(row["alpha01_gamma_residual"]), 6), "V": round(float(row["alpha01_weighted_miscoverage_V"]), 6), + "endpoint_budget": round(float(row["alpha01_endpoint_budget"]), 9), "endpoint_budget_upper": round(float(row["alpha01_endpoint_budget_upper"]), 9), + "Markov_threshold": round(float(row["alpha01_markov_loss_threshold"]), 9), "Markov_cap": round(float(row["alpha01_markov_loss_cap"]), 9), "alpha_pass": f"{int(row['alpha_exact_pass_count'])}/{int(row['alpha_exact_check_count'])}", "n_funded_mean": round(float(row["n_funded_mean"]), 3), @@ -87,7 +112,7 @@ def _score_body_candidate(eligible: pd.DataFrame) -> pd.Series: work = eligible.copy() score_specs = { "return_score": ("alpha01_realized_total_return", False), - "bound_score": ("alpha01_markov_loss_cap", True), + "bound_score": ("alpha01_markov_loss_threshold", True), "v_score": ("alpha01_weighted_miscoverage_V", True), } for score_col, (metric_col, inverse) in score_specs.items(): @@ -103,30 +128,35 @@ def _score_body_candidate(eligible: pd.DataFrame) -> pd.Series: 0.40 * work["return_score"] + 0.40 * work["bound_score"] + 0.20 * work["v_score"] ) return work.sort_values( - ["ijds_balanced_score", "alpha01_realized_total_return", "alpha01_markov_loss_cap"], + [ + "ijds_balanced_score", + "alpha01_realized_total_return", + "alpha01_markov_loss_threshold", + ], ascending=[False, False, True], ).iloc[0] -def _body_candidate(eligible: pd.DataFrame, *, markov_cap: float) -> pd.Series: +def _body_candidate(eligible: pd.DataFrame, *, markov_threshold: float) -> pd.Series: """Select the paper-body point from the exact finite-grid frontier. The body point is intentionally not the global max-return endpoint and not the minimum-bound endpoint. It is the highest-return policy below a declared - Markov-cap ceiling, which matches the paper-facing return-bound claim. + exact Markov-loss threshold, which matches the paper-facing return-bound + claim and remains valid for linear, capped and tail-focused policies. """ - under_cap = _best_under_cap(eligible, markov_cap) - if under_cap is not None: - return under_cap + under_threshold = _best_under_threshold(eligible, markov_threshold) + if under_threshold is not None: + return under_threshold return _score_body_candidate(eligible) -def _best_under_cap(eligible: pd.DataFrame, cap: float) -> pd.Series | None: - candidates = eligible[eligible["alpha01_markov_loss_cap"] <= cap] +def _best_under_threshold(eligible: pd.DataFrame, threshold: float) -> pd.Series | None: + candidates = eligible[eligible["alpha01_markov_loss_threshold"] <= threshold] if candidates.empty: return None return candidates.sort_values( - ["alpha01_realized_total_return", "alpha01_markov_loss_cap"], + ["alpha01_realized_total_return", "alpha01_markov_loss_threshold"], ascending=[False, True], ).iloc[0] @@ -145,9 +175,56 @@ def _load_leaderboards(run_tags: list[str]) -> pd.DataFrame: frames: list[pd.DataFrame] = [] for run_tag in run_tags: path = _leaderboard_path(run_tag) + bound_path = _bound_eval_path(run_tag) if not path.exists(): raise FileNotFoundError(path) + if not bound_path.exists(): + raise FileNotFoundError(bound_path) frame = pd.read_parquet(path) + bound_eval = pd.read_parquet(bound_path) + alpha01 = add_policy_aware_bound_columns( + bound_eval[np.isclose(bound_eval["alpha"], 0.01)].copy() + ) + alpha01_columns = { + "gamma_internalized": "alpha01_gamma_internalized", + "gamma_residual": "alpha01_gamma_residual", + "weighted_pd_constraint_used": "alpha01_weighted_pd_constraint_used", + "weighted_pd_high": "alpha01_weighted_pd_high", + "weighted_pd_point": "alpha01_weighted_pd_point", + "endpoint_budget": "alpha01_endpoint_budget", + "endpoint_budget_upper": "alpha01_endpoint_budget_upper", + "markov_loss_threshold": "alpha01_markov_loss_threshold", + "markov_loss_cap": "alpha01_markov_loss_cap", + } + alpha01 = alpha01[["local_candidate_id", "semantic_policy_key", *alpha01_columns]].rename( + columns=alpha01_columns + ) + legacy_columns = { + "alpha01_endpoint_budget_upper": "legacy_alpha01_endpoint_budget_upper", + "alpha01_markov_loss_cap": "legacy_alpha01_markov_loss_cap", + } + frame = frame.rename( + columns={ + source: target + for source, target in legacy_columns.items() + if source in frame.columns + } + ) + stale_columns = [ + column + for column in alpha01_columns.values() + if column in frame.columns and column not in legacy_columns.values() + ] + if stale_columns: + frame = frame.drop(columns=stale_columns) + frame = frame.merge( + alpha01, + on=["local_candidate_id", "semantic_policy_key"], + how="left", + validate="one_to_one", + ) + if frame["alpha01_markov_loss_threshold"].isna().any(): + raise ValueError(f"Missing alpha=0.01 certificate rows for {run_tag}") frame["run_tag"] = run_tag frame["run_label"] = _short_run_label(run_tag) frames.append(frame) @@ -162,7 +239,7 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ "semantic_policy_key", "alpha_exact_pass_count", "alpha01_realized_total_return", - "alpha01_markov_loss_cap", + "alpha01_markov_loss_threshold", ], ascending=[True, False, False, True], ) @@ -171,10 +248,44 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ ) eligible = _eligible(deduped) - body = _body_candidate(eligible, markov_cap=DEFAULT_BODY_MARKOV_CAP) + body = _body_candidate(eligible, markov_threshold=DEFAULT_BODY_MARKOV_THRESHOLD) + semantics_audit: dict[str, Any] = { + "status": "not_available", + "selection_metric": "alpha01_markov_loss_threshold", + } + if "legacy_alpha01_markov_loss_cap" in deduped.columns: + legacy = pd.to_numeric(deduped["legacy_alpha01_markov_loss_cap"], errors="coerce") + exact = pd.to_numeric(deduped["alpha01_markov_loss_threshold"], errors="coerce") + delta = exact - legacy + tolerance = 1e-5 + legacy_eligible = eligible[eligible["legacy_alpha01_markov_loss_cap"] <= 0.35] + legacy_body = legacy_eligible.sort_values( + ["alpha01_realized_total_return", "legacy_alpha01_markov_loss_cap"], + ascending=[False, True], + ).iloc[0] + semantics_audit = { + "status": "corrected_from_existing_exact_bound_evaluations", + "selection_metric": "alpha01_markov_loss_threshold", + "legacy_metric": "tau + (1 - gamma) * Gamma_CP + sqrt(alpha)", + "material_difference_tolerance": tolerance, + "materially_changed_policies": int((delta.abs() > tolerance).sum()), + "materially_understated_policies": int((delta > tolerance).sum()), + "maximum_legacy_understatement": round(float(delta.max()), 9), + "legacy_under_0_50_excluded_by_exact_threshold": int( + ((legacy <= 0.50) & (exact > 0.50)).sum() + ), + "body_selection_unchanged": bool( + str(legacy_body["semantic_policy_key"]) == str(body["semantic_policy_key"]) + ), + "affected_policy_modes": sorted( + deduped.loc[delta > tolerance, "policy_mode"].astype(str).unique().tolist() + ), + } rows: list[dict[str, Any]] = [] _append_row( - rows, eligible.sort_values("alpha01_markov_loss_cap").iloc[0], "minimum Markov-cap endpoint" + rows, + eligible.sort_values("alpha01_markov_loss_threshold").iloc[0], + "minimum Markov-threshold endpoint", ) _append_row(rows, body, "body/default balanced return-bound point") _append_row( @@ -188,13 +299,17 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ _append_row( rows, eligible.sort_values( - ["alpha01_realized_total_return", "alpha01_markov_loss_cap"], + ["alpha01_realized_total_return", "alpha01_markov_loss_threshold"], ascending=[False, True], ).iloc[0], "max-return economic endpoint", ) for cap in caps: - _append_row(rows, _best_under_cap(eligible, cap), f"highest return under cap<={cap:g}") + _append_row( + rows, + _best_under_threshold(eligible, cap), + f"highest return under threshold<={cap:g}", + ) by_run = [] for run_tag, run_df in raw.groupby("run_tag", sort=False): @@ -215,7 +330,9 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ 9, ), "best_return": round(float(run_df["alpha01_realized_total_return"].max()), 6), - "min_markov_cap": round(float(run_df["alpha01_markov_loss_cap"].min()), 9), + "min_markov_threshold": round( + float(run_df["alpha01_markov_loss_threshold"].min()), 9 + ), } ) @@ -231,9 +348,15 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ ), "body_selection": ( f"highest realized return among eligible finite-grid policies with " - f"Markov_cap <= {DEFAULT_BODY_MARKOV_CAP:g}; falls back to the legacy " + f"exact Markov_threshold <= {DEFAULT_BODY_MARKOV_THRESHOLD:g}; " + "falls back to the legacy " "balanced normalized return/bound/V score only if no eligible policy " - "exists under that declared cap" + "exists under that declared threshold" + ), + "bound_semantics": ( + "Markov_threshold = weighted endpoint budget B_u + sqrt(alpha); " + "Markov_cap = tau + residual endpoint premium + solver slack + sqrt(alpha). " + "The exact threshold drives selection." ), "caps": caps, "role_semantics": "finite-grid frontier roles, not continuous optima", @@ -245,6 +368,7 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ "eligible_all_alpha_return_floor_policies": int(len(eligible)), "nonpass_or_below_floor_policies": int(len(deduped) - len(eligible)), }, + "certificate_semantics_audit": semantics_audit, "by_run": by_run, "rows": rows, } diff --git a/scripts/search/build_pool93_ijds_consolidated_governance.py b/scripts/search/build_pool93_ijds_consolidated_governance.py index 108c119..2d06995 100644 --- a/scripts/search/build_pool93_ijds_consolidated_governance.py +++ b/scripts/search/build_pool93_ijds_consolidated_governance.py @@ -9,7 +9,7 @@ from typing import Any ROOT = Path(__file__).resolve().parents[2] -DEFAULT_CONSOLIDATED_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" +DEFAULT_CONSOLIDATED_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" DEFAULT_BODY_ROLE = "body/default balanced return-bound point" @@ -32,8 +32,8 @@ def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: frontier = json.loads(frontier_path.read_text(encoding="utf-8")) rows = list(frontier.get("rows", [])) body = _find_role(rows, body_role) - strict_cap = _find_role(rows, "highest return under cap<=0.345") - low_cap = _find_role(rows, "minimum Markov-cap endpoint") + strict_threshold = _find_role(rows, "highest return under threshold<=0.345") + low_threshold = _find_role(rows, "minimum Markov-threshold endpoint") max_return = _find_role(rows, "max-return economic endpoint") return { "generated_at_utc": datetime.now(tz=UTC).isoformat(), @@ -41,18 +41,19 @@ def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: "source_run_tags": frontier.get("source_run_tags", []), "counts": frontier.get("counts", {}), "selection_rule": frontier.get("selection_rule", {}), + "certificate_semantics_audit": frontier.get("certificate_semantics_audit", {}), "claim_hierarchy": { "status": "final", "paper_body_candidate": body_role, "paper_body_claim": ( "The selected pool93 body point is the highest-return eligible " - "finite-grid policy under the declared Markov-cap body ceiling " + "finite-grid policy under the declared exact Markov-threshold ceiling " "and passes all eight predeclared alpha checks." ), "appendix_frontier_candidates": [ - "minimum Markov-cap endpoint", - "highest return under cap<=0.345", - "highest return under cap<=0.36", + "minimum Markov-threshold endpoint", + "highest return under threshold<=0.345", + "highest return under threshold<=0.36", "max-return economic endpoint", ], "do_not_claim": [ @@ -64,16 +65,17 @@ def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: "promotion_gate": [ "consolidated frontier generated from completed exact runs", "semantic-policy deduplication applied", + "policy-aware endpoint decomposition applied to every alpha=0.01 row", "selected body point passes 8/8 alpha checks", - "zero exact violation at alpha=0.01", + "zero realized risk-tolerance excess at alpha=0.01", "return exceeds declared return floor", - "A35 frontier and A36 funded-set grade audit are regenerated from retained artifacts", + "A35 frontier, A36 funded-set grade audit, and A40 matched baseline are regenerated from retained artifacts", ], }, "selected_candidates": { "paper_body": body, - "strict_cap_leq_0_345": strict_cap, - "minimum_markov_cap_endpoint": low_cap, + "strict_threshold_leq_0_345": strict_threshold, + "minimum_markov_threshold_endpoint": low_threshold, "max_return_economic_endpoint": max_return, }, "paper_artifacts": { @@ -81,6 +83,13 @@ def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: "frontier_table_tex": "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex", "funded_grade_audit_csv": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", "funded_grade_audit_tex": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex", + "point_baseline_csv": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv", + "point_baseline_tex": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex", + "point_baseline_audit": ( + "models/experiments/champion_reopen/" + "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" + "portfolio/pool93_point_pd_baseline_audit.json" + ), }, } diff --git a/scripts/search/build_pool93_ijds_frontier_claim_table.py b/scripts/search/build_pool93_ijds_frontier_claim_table.py index 27d3961..195a0a8 100644 --- a/scripts/search/build_pool93_ijds_frontier_claim_table.py +++ b/scripts/search/build_pool93_ijds_frontier_claim_table.py @@ -6,6 +6,7 @@ import json from datetime import UTC, datetime from pathlib import Path +from typing import Any, cast import pandas as pd @@ -79,6 +80,7 @@ def build_frontier_table( candidate = claim_summary.get(key, {}) if not isinstance(candidate, dict) or "local_candidate_id" not in candidate: continue + candidate = cast(dict[str, Any], candidate) candidate_id = int(candidate["local_candidate_id"]) _append_unique( rows, diff --git a/scripts/search/build_pool93_point_baseline_audit.py b/scripts/search/build_pool93_point_baseline_audit.py new file mode 100644 index 0000000..78ce7ba --- /dev/null +++ b/scripts/search/build_pool93_point_baseline_audit.py @@ -0,0 +1,372 @@ +"""Build a matched point-PD baseline for the selected pool93 decision. + +The audit holds the candidate universe, budget, concentration cap and risk +tolerance fixed. It changes only the decision uncertainty treatment: the +baseline constrains calibrated point PD, while the selected CRPTO allocation +uses its declared effective-PD policy. Outputs are isolated from the frozen +champion and contain no policy search. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import asdict +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from scripts.optimize_portfolio_tradeoff import _parse_percent_series # noqa: E402 +from scripts.search.build_pool93_body_allocation_audit import ( # noqa: E402 + _load_role_row, + _manifest_path, + _policy_from_row, +) +from scripts.validate_alpha_gamma_bound import ( # noqa: E402 + DEFAULT_LGD, + DEFAULT_MAX_CONCENTRATION, + DEFAULT_TIME_LIMIT, + _compute_intervals_at_alpha, + _load_aligned_dataset, +) +from src.optimization.certificate_semantics import ( # noqa: E402 + FundedCertificateMetrics, + compute_funded_certificate_metrics, +) +from src.optimization.portfolio_model import ( # noqa: E402 + optimize_portfolio_allocation, + solution_allocation_vector, +) +from src.utils.script_helpers import resolve_repo_artifact_path, write_table # noqa: E402 + +DEFAULT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" +DEFAULT_ROLE = "body/default balanced return-bound point" +DEFAULT_FRONTIER = ( + ROOT + / "models/experiments/champion_reopen" + / DEFAULT_TAG + / "portfolio/pool93_ijds_consolidated_frontier.json" +) +DEFAULT_BODY_ALLOCATION = ( + ROOT + / "data/processed/experiments/champion_reopen" + / "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" + / "portfolio/pool93_body_allocation_alpha01.parquet" +) +DEFAULT_OUTPUT_DIR = ROOT / "models/experiments/champion_reopen" / DEFAULT_TAG / "portfolio" +DEFAULT_DATA_OUTPUT_DIR = ( + ROOT / "data/processed/experiments/champion_reopen" / DEFAULT_TAG / "portfolio" +) +DEFAULT_TABLE_DIR = ROOT / "reports/crpto/experiments" / DEFAULT_TAG + + +def _realized_return( + allocation: np.ndarray, + loan_amounts: np.ndarray, + int_rates: np.ndarray, + default_flag: np.ndarray, +) -> float: + exposure = allocation * loan_amounts + funded = allocation > 0.01 + contributions = np.where( + funded & (default_flag.astype(int) == 1), + -DEFAULT_LGD * exposure, + np.where(funded, int_rates * exposure, 0.0), + ) + return float(contributions.sum()) + + +def _economic_metrics( + *, + allocation: np.ndarray, + loan_amounts: np.ndarray, + int_rates: np.ndarray, + pd_point: np.ndarray, +) -> dict[str, float]: + exposure = allocation * loan_amounts + return { + "total_allocated": float(exposure.sum()), + "expected_return_gross": float(np.sum(exposure * int_rates)), + "expected_loss_point": float(np.sum(exposure * pd_point * DEFAULT_LGD)), + "expected_return_net_point": float( + np.sum(exposure * int_rates) - np.sum(exposure * pd_point * DEFAULT_LGD) + ), + } + + +def _comparison_table( + point: dict[str, Any], + selected: dict[str, Any], +) -> pd.DataFrame: + point_return = float(point["realized_return"]) + rows = [] + for label, payload in ( + ("Point-PD two-stage LP", point), + ("Selected CRPTO", selected), + ): + rows.append( + { + "policy": label, + "realized_return": float(payload["realized_return"]), + "return_cost_vs_point_pct": 100.0 + * (point_return - float(payload["realized_return"])) + / point_return, + "n_funded": int(payload["certificate"]["n_funded"]), + "weighted_default_rate": float(payload["certificate"]["weighted_outcome"]), + "V_alpha01": float(payload["certificate"]["weighted_miscoverage"]), + "Gamma_CP_alpha01": float(payload["certificate"]["gamma_cp"]), + "endpoint_budget_alpha01": float(payload["certificate"]["endpoint_budget"]), + "Markov_threshold_alpha01": float(payload["certificate"]["markov_loss_threshold"]), + "expected_return_net_point": float(payload["expected_return_net_point"]), + } + ) + return pd.DataFrame(rows) + + +def _format_comparison_tex(table: pd.DataFrame) -> str: + """Render the compact comparison used in the paper and supplement.""" + lines = [ + "\\begin{tabular}{lrrrrr}", + "\\toprule", + ( + "Policy & Realized return & Weighted default & " + "$\\Gamma_{\\mathrm{CP}}$ & $B_u$ & Markov threshold \\\\" + ), + "\\midrule", + ] + for row in table.to_dict("records"): + lines.append( + f"{row['policy']} & \\${float(row['realized_return']):,.2f} & " + f"{float(row['weighted_default_rate']):.6f} & " + f"{float(row['Gamma_CP_alpha01']):.6f} & " + f"{float(row['endpoint_budget_alpha01']):.6f} & " + f"{float(row['Markov_threshold_alpha01']):.6f} \\\\" + ) + lines.extend(["\\bottomrule", "\\end{tabular}", ""]) + return "\n".join(lines) + + +def _certificate_payload( + certificate: FundedCertificateMetrics, + *, + realized_return: float, + economic: dict[str, float], + solver_status: str, +) -> dict[str, Any]: + return { + "solver_status": solver_status, + "realized_return": realized_return, + **economic, + "certificate": asdict(certificate), + } + + +def build_audit( + *, + frontier_path: Path, + body_allocation_path: Path, + role: str, + alpha: float, + output_dir: Path, + data_output_dir: Path, + table_dir: Path, + threads: int, +) -> dict[str, Any]: + row = _load_role_row(frontier_path, role) + policy = _policy_from_row(row) + manifest = json.loads(_manifest_path(str(row["run_tag"])).read_text(encoding="utf-8")) + interval_path = resolve_repo_artifact_path(manifest["conformal_intervals_path"], root=ROOT) + aligned = _load_aligned_dataset( + conformal_intervals_path=str(interval_path), + max_candidates=int(manifest.get("max_candidates", 0) or 0), + random_state=int(manifest.get("random_state", 42)), + ) + pd_point, pd_low, pd_high = _compute_intervals_at_alpha(aligned, alpha) + y_true = pd.to_numeric(aligned["y_true"], errors="coerce").fillna(0.0).to_numpy(float) + default_flag = ( + pd.to_numeric(aligned["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=int) + ) + loan_amounts = pd.to_numeric(aligned["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(float) + int_rates = _parse_percent_series(aligned["int_rate"]) + risk_tolerance = float(policy["risk_tolerance"]) + solution = optimize_portfolio_allocation( + loans=aligned, + pd_point=pd_point, + pd_low=pd_low, + pd_high=pd_high, + lgd=np.full(len(aligned), DEFAULT_LGD, dtype=float), + int_rates=int_rates, + total_budget=float(manifest.get("budget", 1_000_000.0)), + max_concentration=DEFAULT_MAX_CONCENTRATION, + max_portfolio_pd=risk_tolerance, + robust=False, + uncertainty_aversion=0.0, + min_budget_utilization=float(policy["min_budget_utilization"]), + pd_cap_slack_penalty=float(policy["pd_cap_slack_penalty"]), + pd_constraint_override=pd_point, + time_limit=DEFAULT_TIME_LIMIT, + threads=max(1, int(threads)), + solver_backend=str(policy["solver_backend"]), + ) + point_allocation = solution_allocation_vector(solution, len(aligned)) + point_exposure = point_allocation * loan_amounts + point_weights = point_exposure / max(float(point_exposure.sum()), 1e-12) + point_certificate = compute_funded_certificate_metrics( + point_weights, + outcomes=y_true, + pd_point=pd_point, + pd_high=pd_high, + pd_effective=pd_point, + alpha=alpha, + risk_tolerance=risk_tolerance, + pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), + ) + point_payload = _certificate_payload( + point_certificate, + realized_return=_realized_return( + point_allocation, + loan_amounts, + int_rates, + default_flag, + ), + economic=_economic_metrics( + allocation=point_allocation, + loan_amounts=loan_amounts, + int_rates=int_rates, + pd_point=pd_point, + ), + solver_status=str(solution.get("solver_status", "unknown")), + ) + + funded = pd.read_parquet(body_allocation_path) + selected_weights = funded["funded_weight"].to_numpy(float) + selected_point = funded["pd_point_alpha01"].to_numpy(float) + selected_high = funded["pd_high_alpha01"].to_numpy(float) + selected_effective = funded["effective_pd"].to_numpy(float) + selected_outcomes = funded["default_flag"].to_numpy(float) + selected_exposure = funded["funded_exposure"].to_numpy(float) + selected_rates = _parse_percent_series(funded["int_rate"]) + selected_certificate = compute_funded_certificate_metrics( + selected_weights, + outcomes=selected_outcomes, + pd_point=selected_point, + pd_high=selected_high, + pd_effective=selected_effective, + alpha=alpha, + risk_tolerance=risk_tolerance, + ) + selected_payload = _certificate_payload( + selected_certificate, + realized_return=float(funded["realized_return"].sum()), + economic={ + "total_allocated": float(selected_exposure.sum()), + "expected_return_gross": float(np.sum(selected_exposure * selected_rates)), + "expected_loss_point": float(np.sum(selected_exposure * selected_point * DEFAULT_LGD)), + "expected_return_net_point": float( + np.sum(selected_exposure * selected_rates) + - np.sum(selected_exposure * selected_point * DEFAULT_LGD) + ), + }, + solver_status="frozen_selected_allocation", + ) + + table = _comparison_table(point_payload, selected_payload) + table_paths = write_table( + "crpto_tableA40_pool93_point_baseline", + table, + table_dir=table_dir, + root=ROOT, + float_precision=6, + ) + table_paths[1].write_text( + _format_comparison_tex(table), + encoding="utf-8", + newline="", + ) + output_dir.mkdir(parents=True, exist_ok=True) + data_output_dir.mkdir(parents=True, exist_ok=True) + point_rows = aligned.loc[point_allocation > 0.01].copy() + point_rows["allocation"] = point_allocation[point_allocation > 0.01] + point_rows["funded_exposure"] = point_exposure[point_allocation > 0.01] + point_rows["funded_weight"] = point_weights[point_allocation > 0.01] + point_rows_path = data_output_dir / "pool93_point_pd_baseline_alpha01.parquet" + point_rows.to_parquet(point_rows_path, index=False) + + return_cost = float(point_payload["realized_return"]) - float( + selected_payload["realized_return"] + ) + payload = { + "schema_version": "2026-07-09.1", + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "run_tag": DEFAULT_TAG, + "comparison": "matched point-PD two-stage LP versus selected CRPTO", + "fixed_design": { + "candidate_universe": len(aligned), + "budget": float(manifest.get("budget", 1_000_000.0)), + "risk_tolerance": risk_tolerance, + "max_concentration": DEFAULT_MAX_CONCENTRATION, + "alpha": alpha, + }, + "point_pd_baseline": point_payload, + "selected_crpto": selected_payload, + "contrasts": { + "realized_return_cost": return_cost, + "realized_return_cost_pct": 100.0 + * return_cost + / float(point_payload["realized_return"]), + "weighted_default_rate_reduction": point_certificate.weighted_outcome + - selected_certificate.weighted_outcome, + "weighted_miscoverage_reduction": point_certificate.weighted_miscoverage + - selected_certificate.weighted_miscoverage, + "markov_threshold_reduction": point_certificate.markov_loss_threshold + - selected_certificate.markov_loss_threshold, + }, + "claim_boundary": ( + "Frozen OOT matched-policy audit; it quantifies a return-risk trade-off and " + "does not establish causal, prospective or universal dominance." + ), + "outputs": { + "point_funded_rows": str(point_rows_path), + "table_csv": str(table_paths[0]), + "table_tex": str(table_paths[1]), + }, + } + output_path = output_dir / "pool93_point_pd_baseline_audit.json" + output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8", newline="") + return payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier", default=str(DEFAULT_FRONTIER)) + parser.add_argument("--body-allocation", default=str(DEFAULT_BODY_ALLOCATION)) + parser.add_argument("--role", default=DEFAULT_ROLE) + parser.add_argument("--alpha", type=float, default=0.01) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) + parser.add_argument("--data-output-dir", default=str(DEFAULT_DATA_OUTPUT_DIR)) + parser.add_argument("--table-dir", default=str(DEFAULT_TABLE_DIR)) + args = parser.parse_args(argv) + payload = build_audit( + frontier_path=Path(args.frontier).resolve(), + body_allocation_path=Path(args.body_allocation).resolve(), + role=str(args.role), + alpha=float(args.alpha), + output_dir=Path(args.output_dir).resolve(), + data_output_dir=Path(args.data_output_dir).resolve(), + table_dir=Path(args.table_dir).resolve(), + threads=max(1, int(args.threads)), + ) + print(json.dumps(payload["contrasts"], indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/search/export_pool93_policy_aware_frontier.py b/scripts/search/export_pool93_policy_aware_frontier.py new file mode 100644 index 0000000..17e505b --- /dev/null +++ b/scripts/search/export_pool93_policy_aware_frontier.py @@ -0,0 +1,143 @@ +"""Export the policy-aware pool93 frontier as CSV and LaTeX.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import pandas as pd + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +DEFAULT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" +DEFAULT_FRONTIER = ( + ROOT + / "models/experiments/champion_reopen" + / DEFAULT_TAG + / "portfolio/pool93_ijds_consolidated_frontier.json" +) +DEFAULT_TABLE_DIR = ROOT / "reports/crpto/experiments" / DEFAULT_TAG +DEFAULT_TABLE_NAME = "crpto_tableA35_pool93_ijds_frontier_policy_aware_v2" + +ROLE_ORDER = ( + "minimum Markov-threshold endpoint", + "lowest realized V return-bound point", + "highest return under threshold<=0.3", + "highest return under threshold<=0.32", + "highest return under threshold<=0.345", + "body/default balanced return-bound point", + "highest return under threshold<=0.36", + "highest return under threshold<=0.45", + "max-return economic endpoint", +) +ROLE_LABELS = { + "minimum Markov-threshold endpoint": "Minimum Markov-threshold endpoint", + "lowest realized V return-bound point": "Low-threshold balanced endpoint", + "highest return under threshold<=0.3": "Highest return under threshold <= 0.30", + "highest return under threshold<=0.32": "Highest return under threshold <= 0.32", + "highest return under threshold<=0.345": "Highest return under threshold <= 0.345", + "body/default balanced return-bound point": "Body/default balanced point", + "highest return under threshold<=0.36": "Highest return under threshold <= 0.36", + "highest return under threshold<=0.45": "Highest return under threshold <= 0.45", + "max-return economic endpoint": "Max-return economic endpoint", +} + + +def build_table(frontier: dict[str, Any]) -> pd.DataFrame: + """Build the compact publication table in its declared role order.""" + rows_by_role = {str(row["role"]): dict(row) for row in frontier.get("rows", [])} + missing = [role for role in ROLE_ORDER if role not in rows_by_role] + if missing: + raise ValueError(f"Policy-aware frontier is missing roles: {missing}") + rows = [] + for role in ROLE_ORDER: + row = rows_by_role[role] + rows.append( + { + "role": ROLE_LABELS[role], + "source_run": str(row["run_label"]), + "candidate_id": int(row["local_candidate_id"]), + "policy_family": str(row["family"]), + "risk_tolerance": float(row["risk_tolerance"]), + "policy_mode": str(row["policy_mode"]), + "gamma": float(row["gamma"]), + "uncertainty_aversion": float(row["uncertainty_aversion"]), + "realized_return": float(row["return"]), + "return_floor_surplus": float(row["return_floor_surplus"]), + "Gamma_CP_alpha01": float(row["Gamma_CP"]), + "Gamma_residual_alpha01": float(row["Gamma_residual"]), + "V_alpha01": float(row["V"]), + "endpoint_budget_alpha01": float(row["endpoint_budget"]), + "endpoint_budget_upper_alpha01": float(row["endpoint_budget_upper"]), + "Markov_threshold_alpha01": float(row["Markov_threshold"]), + "Markov_cap_alpha01": float(row["Markov_cap"]), + "alpha_grid_pass": str(row["alpha_pass"]), + "n_funded_mean": float(row["n_funded_mean"]), + } + ) + return pd.DataFrame(rows) + + +def _format_tex(table: pd.DataFrame) -> str: + lines = [ + "\\begin{tabular}{llrrrrrr}", + "\\toprule", + ( + "Role & Source & Return & $\\Gamma_{\\mathrm{CP}}$ & " + "$\\Gamma_{\\mathrm{res}}$ & $V$ & Markov threshold & Pass \\\\" + ), + "\\midrule", + ] + for row in table.to_dict("records"): + role = str(row["role"]).replace("<=", "$\\leq$") + source = str(row["source_run"]).replace("_", " ") + lines.append( + f"{role} & {source} & {float(row['realized_return']):,.2f} & " + f"{float(row['Gamma_CP_alpha01']):.6f} & " + f"{float(row['Gamma_residual_alpha01']):.6f} & " + f"{float(row['V_alpha01']):.6f} & " + f"{float(row['Markov_threshold_alpha01']):.6f} & " + f"{row['alpha_grid_pass']} \\\\" + ) + lines.extend(["\\bottomrule", "\\end{tabular}", ""]) + return "\n".join(lines) + + +def write_frontier_table(table: pd.DataFrame, *, table_dir: Path, table_name: str) -> None: + """Write the full audit CSV and compact publication LaTeX table.""" + table_dir.mkdir(parents=True, exist_ok=True) + csv_path = table_dir / f"{table_name}.csv" + tex_path = table_dir / f"{table_name}.tex" + outputs = { + csv_path: table.to_csv(index=False, lineterminator="\n"), + tex_path: _format_tex(table), + } + for path, content in outputs.items(): + if path.exists() and path.read_text(encoding="utf-8") == content: + continue + path.write_text(content, encoding="utf-8", newline="") + print(f"Wrote {csv_path.relative_to(ROOT).as_posix()}") + print(f"Wrote {tex_path.relative_to(ROOT).as_posix()}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier", default=str(DEFAULT_FRONTIER)) + parser.add_argument("--table-dir", default=str(DEFAULT_TABLE_DIR)) + parser.add_argument("--table-name", default=DEFAULT_TABLE_NAME) + args = parser.parse_args(argv) + frontier = json.loads(Path(args.frontier).read_text(encoding="utf-8")) + write_frontier_table( + build_table(frontier), + table_dir=Path(args.table_dir).resolve(), + table_name=str(args.table_name), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/search/run_conformal_reopen_search.py b/scripts/search/run_conformal_reopen_search.py index ee4b8a4..38d3fc4 100644 --- a/scripts/search/run_conformal_reopen_search.py +++ b/scripts/search/run_conformal_reopen_search.py @@ -9,6 +9,7 @@ import subprocess import sys from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -33,6 +34,57 @@ from src.models.venn_abers import VennAbersScoreCalibrator from src.utils.pipeline_topology import load_profile_config + +@dataclass(frozen=True) +class Phase1Result: + shortlist: pd.DataFrame + aggregated: pd.DataFrame + inner_runs: list[dict[str, Any]] + aggregate_path: str + inner_search_path: str + inner_search_winner: dict[str, Any] + resume_meta: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class Phase1ConfirmationResult: + candidates: list[dict[str, Any]] + frame: pd.DataFrame + best_namespace: str + final_policy: dict[str, Any] + final_sets: dict[str, Any] + final_decision: str + final_namespace: str + + +@dataclass(frozen=True) +class PromotionResult: + final_policy: dict[str, Any] + final_sets: dict[str, Any] + final_decision: str + final_namespace: str + phase2_summary: dict[str, Any] | None + + +@dataclass +class Phase2SearchState: + paths: dict[str, Path] + run_tag: str + upstream_run_tag: str + total_design_runs: int + design_source: str + completed: list[dict[str, Any]] + skipped: list[dict[str, Any]] + failed: list[dict[str, Any]] + + +@dataclass(frozen=True) +class Phase2CalibratorFit: + method: str + path: Path + metrics: dict[str, float] + + REPO_ROOT = Path(__file__).resolve().parents[2] @@ -915,6 +967,301 @@ def _build_resume_shortlist( return shortlist, resume_meta +def _phase2_methods(phase2_cfg: dict[str, Any]) -> list[str]: + return [ + str(method).strip().lower() + for method in phase2_cfg.get("calibrators", ["venn_abers", "isotonic", "platt", "beta"]) + if str(method).strip() + ] + + +def _write_phase2_state_progress( + state: Phase2SearchState, + running: list[dict[str, Any]], +) -> None: + _write_phase2_progress( + path=state.paths["phase2_progress"], + run_tag=state.run_tag, + upstream_run_tag=state.upstream_run_tag, + total_design_runs=state.total_design_runs, + completed=state.completed, + running=running, + skipped=state.skipped, + failed=state.failed, + design_source=state.design_source, + ) + + +def _phase2_baseline_metrics( + *, + calibrator_dir: Path, + upstream_run_tag: str, +) -> dict[str, float] | None: + baseline_path = calibrator_dir / "venn_abers.pkl" + try: + _resolved, baseline_metrics = _fit_calibrator( + method="venn_abers", + output_path=baseline_path, + upstream_run_tag=upstream_run_tag, + ) + except Exception: + return None + return baseline_metrics + + +def _numeric_metric_payload(metrics: dict[str, float]) -> dict[str, float]: + return {key: float(value) for key, value in metrics.items() if isinstance(value, int | float)} + + +def _phase2_metric_blocked( + *, + calibration_metrics: dict[str, float], + baseline_metrics: dict[str, float] | None, + max_metric_degradation: dict[str, Any], +) -> bool: + if baseline_metrics is None: + return False + return any( + float(calibration_metrics.get(metric_name, float("inf"))) + > float(baseline_metrics.get(metric_name, 0.0)) + + float(max_metric_degradation.get(metric_name, 0.0)) + for metric_name in max_metric_degradation + ) + + +def _fit_phase2_calibrator_or_skip( + *, + method_name: str, + calibrator_dir: Path, + state: Phase2SearchState, + baseline_metrics: dict[str, float] | None, + max_metric_degradation: dict[str, Any], +) -> Phase2CalibratorFit | None: + calibrator_path = calibrator_dir / f"{method_name}.pkl" + _write_phase2_state_progress( + state, + running=[{"stage": "fit_calibrator", "calibrator_method": method_name}], + ) + try: + resolved_method, calibration_metrics = _fit_calibrator( + method=method_name, + output_path=calibrator_path, + upstream_run_tag=state.upstream_run_tag, + ) + except Exception as exc: + state.failed.append( + { + "stage": "fit_calibrator", + "calibrator_method": method_name, + "error": repr(exc), + } + ) + _write_phase2_state_progress(state, running=[]) + raise + + if _phase2_metric_blocked( + calibration_metrics=calibration_metrics, + baseline_metrics=baseline_metrics, + max_metric_degradation=max_metric_degradation, + ): + state.skipped.append( + { + "stage": "calibrator_gate", + "calibrator_method": resolved_method, + "reason": "metric_degradation_gate", + "calibration_metrics": _numeric_metric_payload(calibration_metrics), + } + ) + _write_phase2_state_progress(state, running=[]) + return None + + return Phase2CalibratorFit( + method=resolved_method, + path=calibrator_path, + metrics=calibration_metrics, + ) + + +def _phase2_candidate_namespace( + *, + run_tag: str, + calibrator_method: str, + design: dict[str, Any], +) -> str: + return _namespace( + run_tag, + "phase2", + calibrator_method, + f"rank-{int(design.get('selection_rank', 1))}", + ) + + +def _run_phase2_holdout_candidate( + *, + state: Phase2SearchState, + env: dict[str, str], + calibrator_fit: Phase2CalibratorFit, + design: dict[str, Any], + alpha_candidates_95: list[float], + tuning_holdout_ratios: list[float], + inner_random_states: list[int], +) -> dict[str, Any]: + design_norm = _normalize_design_row(design) + selection_rank = int(design.get("selection_rank", 1)) + namespace = _phase2_candidate_namespace( + run_tag=state.run_tag, + calibrator_method=calibrator_fit.method, + design=design, + ) + running_entry = { + "stage": "generate_intervals", + "artifact_namespace": namespace, + "calibrator_method": calibrator_fit.method, + "selection_rank": selection_rank, + } + _write_phase2_state_progress(state, running=[running_entry]) + try: + _run_python( + "scripts/generate_conformal_intervals.py", + [ + "--artifact_namespace", + namespace, + "--evaluation_scope", + "holdout", + "--calibrator_override_path", + str(calibrator_fit.path), + "--tuning_holdout_ratio", + str(float(tuning_holdout_ratios[0])), + "--tuning_random_state", + str(int(inner_random_states[0])), + "--alpha_candidates_95", + ",".join(str(x) for x in alpha_candidates_95), + *_design_args(design_norm), + ], + env, + ) + except Exception as exc: + state.failed.append({**running_entry, "error": repr(exc)}) + _write_phase2_state_progress(state, running=[]) + raise + + payload = _load_pickle(_resolve_run_paths(namespace)["results"]) + metrics_90 = dict(payload.get("metrics_90", {}) or {}) + state.completed.append( + { + "artifact_namespace": namespace, + "calibrator_method": calibrator_fit.method, + "selection_rank": selection_rank, + "coverage_90": float(metrics_90.get("empirical_coverage", 0.0)), + "avg_width_90": float(metrics_90.get("avg_interval_width", 1.0)), + } + ) + _write_phase2_state_progress(state, running=[]) + return { + "artifact_namespace": namespace, + "calibrator_method": calibrator_fit.method, + "phase2_design_source": state.design_source, + "selection_rank": selection_rank, + **design_norm, + "holdout_coverage": float(metrics_90.get("empirical_coverage", 0.0)), + "holdout_width": float(metrics_90.get("avg_interval_width", 1.0)), + "calibrator_ece": float(calibrator_fit.metrics.get("ece", float("inf"))), + "calibrator_adaptive_ece": float(calibrator_fit.metrics.get("adaptive_ece", float("inf"))), + "calibrator_brier": float(calibrator_fit.metrics.get("brier_score", float("inf"))), + "calibrator_log_loss": float(calibrator_fit.metrics.get("log_loss", float("inf"))), + "calibrator_phi_brier": float(calibrator_fit.metrics.get("phi_brier_score", 0.0)), + "calibrator_phi_log_loss": float(calibrator_fit.metrics.get("phi_log_loss", 0.0)), + } + + +def _empty_phase2_frame() -> pd.DataFrame: + return pd.DataFrame( + { + "artifact_namespace": pd.Series(dtype="object"), + "calibrator_method": pd.Series(dtype="object"), + "phase2_design_source": pd.Series(dtype="object"), + "selection_rank": pd.Series(dtype="int64"), + "holdout_coverage": pd.Series(dtype="float64"), + "holdout_width": pd.Series(dtype="float64"), + "calibrator_ece": pd.Series(dtype="float64"), + "calibrator_adaptive_ece": pd.Series(dtype="float64"), + "calibrator_brier": pd.Series(dtype="float64"), + "calibrator_log_loss": pd.Series(dtype="float64"), + "calibrator_phi_brier": pd.Series(dtype="float64"), + "calibrator_phi_log_loss": pd.Series(dtype="float64"), + } + ) + + +def _rank_phase2_candidates(phase2_df: pd.DataFrame) -> pd.DataFrame: + ranked = phase2_df.copy() + ranked["coverage_gap_abs"] = (ranked["holdout_coverage"] - 0.90).abs() + return ranked.sort_values( + by=[ + "coverage_gap_abs", + "holdout_width", + "calibrator_ece", + "calibrator_adaptive_ece", + "calibrator_brier", + "calibrator_phi_brier", + "selection_rank", + ], + ascending=[True, True, True, True, True, False, True], + ).reset_index(drop=True) + + +def _phase2_no_candidate_result( + *, + paths: dict[str, Path], + design_source: str, +) -> tuple[str, dict[str, Any], dict[str, Any], dict[str, Any]]: + return ( + "policy_review_candidate", + {}, + {}, + { + "search_path": str(paths["phase2_search"]), + "best_candidate": {}, + "status": "no_noninferior_calibrator_candidate", + "phase2_design_source": design_source, + }, + ) + + +def _run_phase2_final_candidate( + *, + run_tag: str, + env: dict[str, str], + calibrator_dir: Path, + phase2_best: dict[str, Any], + alpha_candidates_95: list[float], + partition_candidates: list[str], + partition_probability_sources: list[str], + n_score_bins_candidates: list[int], + fallback_modes: list[str], + score_scale_families: list[str], + calibration_fractions: list[float], + sidecar_cfg: dict[str, Any], +) -> dict[str, Any]: + phase2_calibrator_path = calibrator_dir / f"{phase2_best['calibrator_method']}.pkl" + return _run_phase1_oot_candidate( + run_tag=run_tag, + rank=1, + design=phase2_best, + env=env, + alpha_candidates_95=alpha_candidates_95, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + calibration_fractions=calibration_fractions, + sidecar_cfg=sidecar_cfg, + calibrator_override_path=str(phase2_calibrator_path), + phase_prefix="phase2", + ) + + def _run_phase2_search( *, run_tag: str, @@ -938,270 +1285,72 @@ def _run_phase2_search( paths = _reopen_artifact_paths(run_tag) models_dir = paths["models_dir"] calibrator_dir = models_dir / "phase2_calibrators" - calibrator_rows: list[dict[str, Any]] = [] top_designs, design_source = _phase2_top_designs( aggregated=aggregated, phase1_candidates_frame=phase1_candidates_frame, top_k=int(phase2_cfg.get("top_k_designs", 3)), ) - phase2_methods = [ - str(method).strip().lower() - for method in phase2_cfg.get("calibrators", ["venn_abers", "isotonic", "platt", "beta"]) - if str(method).strip() - ] - phase2_completed: list[dict[str, Any]] = [] - phase2_skipped: list[dict[str, Any]] = [] - phase2_failed: list[dict[str, Any]] = [] + phase2_methods = _phase2_methods(phase2_cfg) total_design_runs = len(phase2_methods) * len(top_designs) - _write_phase2_progress( - path=paths["phase2_progress"], + state = Phase2SearchState( + paths=paths, run_tag=run_tag, upstream_run_tag=upstream_run_tag, total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, design_source=design_source, + completed=[], + skipped=[], + failed=[], + ) + _write_phase2_state_progress(state, running=[]) + baseline_metrics = _phase2_baseline_metrics( + calibrator_dir=calibrator_dir, + upstream_run_tag=upstream_run_tag, ) - baseline_metrics: dict[str, float] | None = None - baseline_path = calibrator_dir / "venn_abers.pkl" - try: - _resolved, baseline_metrics = _fit_calibrator( - method="venn_abers", - output_path=baseline_path, - upstream_run_tag=upstream_run_tag, - ) - except Exception: - baseline_metrics = None max_metric_degradation = dict(phase2_cfg.get("max_metric_degradation", {}) or {}) + calibrator_rows: list[dict[str, Any]] = [] for method_name in phase2_methods: - calibrator_path = calibrator_dir / f"{method_name}.pkl" - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[{"stage": "fit_calibrator", "calibrator_method": method_name}], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, + calibrator_fit = _fit_phase2_calibrator_or_skip( + method_name=method_name, + calibrator_dir=calibrator_dir, + state=state, + baseline_metrics=baseline_metrics, + max_metric_degradation=max_metric_degradation, ) - try: - resolved_method, calibration_metrics = _fit_calibrator( - method=method_name, - output_path=calibrator_path, - upstream_run_tag=upstream_run_tag, - ) - except Exception as exc: - phase2_failed.append( - { - "stage": "fit_calibrator", - "calibrator_method": method_name, - "error": repr(exc), - } - ) - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) - raise - if baseline_metrics is not None: - metric_blocked = any( - float(calibration_metrics.get(metric_name, float("inf"))) - > float(baseline_metrics.get(metric_name, 0.0)) - + float(max_metric_degradation.get(metric_name, 0.0)) - for metric_name in max_metric_degradation - ) - if metric_blocked: - phase2_skipped.append( - { - "stage": "calibrator_gate", - "calibrator_method": resolved_method, - "reason": "metric_degradation_gate", - "calibration_metrics": { - key: float(value) - for key, value in calibration_metrics.items() - if isinstance(value, int | float) - }, - } - ) - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) - continue + if calibrator_fit is None: + continue for design in top_designs.to_dict(orient="records"): - design_norm = _normalize_design_row(design) - namespace = _namespace( - run_tag, - "phase2", - resolved_method, - f"rank-{int(design.get('selection_rank', 1))}", - ) - running_entry = { - "stage": "generate_intervals", - "artifact_namespace": namespace, - "calibrator_method": resolved_method, - "selection_rank": int(design.get("selection_rank", 1)), - } - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[running_entry], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) - try: - _run_python( - "scripts/generate_conformal_intervals.py", - [ - "--artifact_namespace", - namespace, - "--evaluation_scope", - "holdout", - "--calibrator_override_path", - str(calibrator_path), - "--tuning_holdout_ratio", - str(float(tuning_holdout_ratios[0])), - "--tuning_random_state", - str(int(inner_random_states[0])), - "--alpha_candidates_95", - ",".join(str(x) for x in alpha_candidates_95), - *_design_args(design_norm), - ], - env, - ) - except Exception as exc: - phase2_failed.append({**running_entry, "error": repr(exc)}) - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) - raise - payload = _load_pickle(_resolve_run_paths(namespace)["results"]) - metrics_90 = dict(payload.get("metrics_90", {}) or {}) - phase2_completed.append( - { - "artifact_namespace": namespace, - "calibrator_method": resolved_method, - "selection_rank": int(design.get("selection_rank", 1)), - "coverage_90": float(metrics_90.get("empirical_coverage", 0.0)), - "avg_width_90": float(metrics_90.get("avg_interval_width", 1.0)), - } - ) - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) calibrator_rows.append( - { - "artifact_namespace": namespace, - "calibrator_method": resolved_method, - "phase2_design_source": design_source, - "selection_rank": int(design.get("selection_rank", 1)), - **design_norm, - "holdout_coverage": float(metrics_90.get("empirical_coverage", 0.0)), - "holdout_width": float(metrics_90.get("avg_interval_width", 1.0)), - "calibrator_ece": float(calibration_metrics.get("ece", float("inf"))), - "calibrator_adaptive_ece": float( - calibration_metrics.get("adaptive_ece", float("inf")) - ), - "calibrator_brier": float(calibration_metrics.get("brier_score", float("inf"))), - "calibrator_log_loss": float(calibration_metrics.get("log_loss", float("inf"))), - "calibrator_phi_brier": float(calibration_metrics.get("phi_brier_score", 0.0)), - "calibrator_phi_log_loss": float(calibration_metrics.get("phi_log_loss", 0.0)), - } + _run_phase2_holdout_candidate( + state=state, + env=env, + calibrator_fit=calibrator_fit, + design=design, + alpha_candidates_95=alpha_candidates_95, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, + ) ) phase2_df = pd.DataFrame(calibrator_rows) if phase2_df.empty: - phase2_df = pd.DataFrame( - columns=[ - "artifact_namespace", - "calibrator_method", - "phase2_design_source", - "selection_rank", - "holdout_coverage", - "holdout_width", - "calibrator_ece", - "calibrator_adaptive_ece", - "calibrator_brier", - "calibrator_log_loss", - "calibrator_phi_brier", - "calibrator_phi_log_loss", - ] - ) + phase2_df = _empty_phase2_frame() phase2_df.to_parquet(paths["phase2_search"], index=False) - return ( - "policy_review_candidate", - {}, - {}, - { - "search_path": str(paths["phase2_search"]), - "best_candidate": {}, - "status": "no_noninferior_calibrator_candidate", - "phase2_design_source": design_source, - }, + return _phase2_no_candidate_result( + paths=paths, + design_source=design_source, ) - phase2_df["coverage_gap_abs"] = (phase2_df["holdout_coverage"] - 0.90).abs() - phase2_df = phase2_df.sort_values( - by=[ - "coverage_gap_abs", - "holdout_width", - "calibrator_ece", - "calibrator_adaptive_ece", - "calibrator_brier", - "calibrator_phi_brier", - "selection_rank", - ], - ascending=[True, True, True, True, True, False, True], - ).reset_index(drop=True) + phase2_df = _rank_phase2_candidates(phase2_df) phase2_df.to_parquet(paths["phase2_search"], index=False) phase2_best = phase2_df.iloc[0].to_dict() - _namespace(run_tag, "phase2", "final") - phase2_calibrator_path = calibrator_dir / f"{phase2_best['calibrator_method']}.pkl" - final_candidate = _run_phase1_oot_candidate( + final_candidate = _run_phase2_final_candidate( run_tag=run_tag, - rank=1, - design=phase2_best, env=env, + calibrator_dir=calibrator_dir, + phase2_best=phase2_best, alpha_candidates_95=alpha_candidates_95, partition_candidates=partition_candidates, partition_probability_sources=partition_probability_sources, @@ -1210,8 +1359,6 @@ def _run_phase2_search( score_scale_families=score_scale_families, calibration_fractions=calibration_fractions, sidecar_cfg=sidecar_cfg, - calibrator_override_path=str(phase2_calibrator_path), - phase_prefix="phase2", ) final_policy = dict(final_candidate["policy_status"]) final_sets = dict(final_candidate["set_status"]) @@ -1274,7 +1421,7 @@ def _write_consolidated_status( _write_json(_reopen_artifact_paths(run_tag)["status"], status) -def main(argv: list[str] | None = None) -> int: +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument("--run-tag", default=_default_run_tag()) parser.add_argument("--pipeline-profile", default="search_conformal_reopen_exhaustive") @@ -1297,103 +1444,139 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Evaluate phase-2 calibrator tournament even when phase-1 already passes.", ) - args = parser.parse_args(argv) + return parser - run_tag = str(args.run_tag).strip() - upstream_run_tag = str(args.upstream_canonical_run_tag).strip() - resume_from_run_tag = str(args.resume_from_run_tag).strip() if args.resume_from_run_tag else "" - mode = "derived_resume" if resume_from_run_tag else "fresh_search" - profile = _profile_cfg(args.pipeline_profile) - cfg = _phase1_cfg(profile) - phase2_cfg = _phase2_cfg(profile) - sidecar_cfg = _sidecar_cfg(profile) - validation_cfg = _validation_cfg(profile) - output_paths = _reopen_artifact_paths(run_tag) +def _phase1_from_resume( + *, + resume_from_run_tag: str, + top_k_inner: int, + output_paths: dict[str, Path], +) -> Phase1Result: + source_paths = _reopen_artifact_paths(resume_from_run_tag) + if not source_paths["inner_aggregate"].exists(): + raise FileNotFoundError( + f"Resume source missing aggregate artifact: {source_paths['inner_aggregate']}" + ) + shortlist, resume_meta = _build_resume_shortlist( + source_run_tag=resume_from_run_tag, + top_k_inner=top_k_inner, + ) + shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) + aggregated = pd.read_parquet(source_paths["inner_aggregate"]) + return Phase1Result( + shortlist=shortlist, + aggregated=aggregated, + inner_runs=[], + aggregate_path=str(source_paths["inner_aggregate"]), + inner_search_path=str(source_paths["inner_search"]), + inner_search_winner=_normalize_design_row(aggregated.iloc[0].to_dict()) + if not aggregated.empty + else {}, + resume_meta=resume_meta, + ) - env = os.environ.copy() - env["PIPELINE_RUN_TAG"] = run_tag - env["UPSTREAM_CANONICAL_RUN_TAG"] = upstream_run_tag - alpha_candidates_90 = cfg.get("alpha_candidates_90", [0.09, 0.095, 0.10, 0.105, 0.11, 0.12]) - alpha_candidates_95 = cfg.get("alpha_candidates_95", [0.045, 0.05, 0.055, 0.06]) - partition_candidates = cfg.get( - "partition_candidates", - ["grade", "score_decile_mondrian", "grade_x_scoreband_mondrian"], +def _run_phase1_inner_search( + *, + run_tag: str, + upstream_run_tag: str, + env: dict[str, str], + output_paths: dict[str, Path], + phase1_workers: int, + top_k_inner: int, + alpha_candidates_90: list[float], + alpha_candidates_95: list[float], + partition_candidates: list[str], + partition_probability_sources: list[str], + n_score_bins_candidates: list[int], + min_group_sizes: list[int], + fallback_modes: list[str], + score_scale_families: list[str], + calibration_fractions: list[float], + tuning_holdout_ratios: list[float], + inner_random_states: list[int], + resume_completed_inner: bool, +) -> Phase1Result: + inner_frames: list[pd.DataFrame] = [] + inner_runs: list[dict[str, Any]] = [] + specs = _phase1_inner_specs( + run_tag=run_tag, + calibration_fractions=calibration_fractions, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, ) - partition_probability_sources = cfg.get("partition_probability_sources", ["calibrated", "raw"]) - n_score_bins_candidates = cfg.get("n_score_bins_candidates", [5, 10, 15, 20]) - min_group_sizes = cfg.get("min_group_sizes", [100, 150, 250, 500, 1000]) - fallback_modes = cfg.get("fallback_modes", ["grade_then_global", "global_only"]) - score_scale_families = cfg.get( - "score_scale_families", - ["none", "bernoulli_sqrt", "bernoulli_sqrt_clipped_0.02", "bernoulli_sqrt_clipped_0.05"], + completed_runs: list[dict[str, Any]] = [] + failed_runs: list[dict[str, Any]] = [] + running: set[str] = set() + progress_path = output_paths["phase1_progress"] + _write_phase1_progress( + path=progress_path, + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + total=len(specs), + completed=completed_runs, + running=[], + failed=failed_runs, + workers=phase1_workers, ) - calibration_fractions = cfg.get("calibration_fractions", [0.25, 0.50, 0.75, 1.00]) - tuning_holdout_ratios = cfg.get("tuning_holdout_ratios", [0.20, 0.30]) - inner_random_states = cfg.get("inner_random_states", [42, 314, 2026]) - top_k_inner = int(validation_cfg.get("top_k_inner", 3)) - configured_workers = int(cfg.get("parallel_workers", 1) or 1) - phase1_workers = int(args.phase1_workers or configured_workers or 1) - phase1_workers = max(1, phase1_workers) - resume_completed_inner = not bool(args.no_resume_completed_inner) - resume_meta: dict[str, Any] | None = None - inner_runs: list[dict[str, Any]] = [] - if resume_from_run_tag: - source_paths = _reopen_artifact_paths(resume_from_run_tag) - if not source_paths["inner_aggregate"].exists(): - raise FileNotFoundError( - f"Resume source missing aggregate artifact: {source_paths['inner_aggregate']}" - ) - shortlist, resume_meta = _build_resume_shortlist( - source_run_tag=resume_from_run_tag, - top_k_inner=top_k_inner, - ) - shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) - aggregate_path = str(source_paths["inner_aggregate"]) - inner_search_path = str(source_paths["inner_search"]) - aggregated = pd.read_parquet(source_paths["inner_aggregate"]) - inner_search_winner = ( - _normalize_design_row(aggregated.iloc[0].to_dict()) if not aggregated.empty else {} - ) - else: - inner_frames: list[pd.DataFrame] = [] - specs = _phase1_inner_specs( - run_tag=run_tag, - calibration_fractions=calibration_fractions, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, - ) - completed_runs: list[dict[str, Any]] = [] - failed_runs: list[dict[str, Any]] = [] - running: set[str] = set() - progress_path = output_paths["phase1_progress"] - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=[], - failed=failed_runs, - workers=phase1_workers, + def record_success(tuning_df: pd.DataFrame, run_summary: dict[str, Any]) -> None: + inner_frames.append(tuning_df) + inner_runs.append(run_summary) + completed_runs.append( + { + "artifact_namespace": run_summary["artifact_namespace"], + "reused_existing": bool(run_summary.get("reused_existing", False)), + } ) - if phase1_workers == 1: + if phase1_workers == 1: + for spec in specs: + running = {str(spec["namespace"])} + _write_phase1_progress( + path=progress_path, + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + total=len(specs), + completed=completed_runs, + running=sorted(running), + failed=failed_runs, + workers=phase1_workers, + ) + tuning_df, run_summary = _run_phase1_inner_spec( + spec=spec, + env=env, + alpha_candidates_90=alpha_candidates_90, + alpha_candidates_95=alpha_candidates_95, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + min_group_sizes=min_group_sizes, + score_scale_families=score_scale_families, + resume_completed=resume_completed_inner, + ) + record_success(tuning_df, run_summary) + running = set() + _write_phase1_progress( + path=progress_path, + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + total=len(specs), + completed=completed_runs, + running=[], + failed=failed_runs, + workers=phase1_workers, + ) + else: + with ThreadPoolExecutor(max_workers=phase1_workers) as executor: + future_to_spec = {} for spec in specs: - running = {str(spec["namespace"])} - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - tuning_df, run_summary = _run_phase1_inner_spec( + namespace = str(spec["namespace"]) + running.add(namespace) + future = executor.submit( + _run_phase1_inner_spec, spec=spec, env=env, alpha_candidates_90=alpha_candidates_90, @@ -1406,86 +1589,28 @@ def main(argv: list[str] | None = None) -> int: score_scale_families=score_scale_families, resume_completed=resume_completed_inner, ) - inner_frames.append(tuning_df) - inner_runs.append(run_summary) - completed_runs.append( - { - "artifact_namespace": run_summary["artifact_namespace"], - "reused_existing": bool(run_summary.get("reused_existing", False)), - } - ) - running = set() + future_to_spec[future] = spec _write_phase1_progress( path=progress_path, run_tag=run_tag, upstream_run_tag=upstream_run_tag, total=len(specs), completed=completed_runs, - running=[], + running=sorted(running), failed=failed_runs, workers=phase1_workers, ) - else: - with ThreadPoolExecutor(max_workers=phase1_workers) as executor: - future_to_spec = {} - for spec in specs: - namespace = str(spec["namespace"]) - running.add(namespace) - future = executor.submit( - _run_phase1_inner_spec, - spec=spec, - env=env, - alpha_candidates_90=alpha_candidates_90, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - min_group_sizes=min_group_sizes, - score_scale_families=score_scale_families, - resume_completed=resume_completed_inner, - ) - future_to_spec[future] = spec - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - for future in as_completed(future_to_spec): - spec = future_to_spec[future] - namespace = str(spec["namespace"]) - running.discard(namespace) - try: - tuning_df, run_summary = future.result() - except Exception as exc: - failed_runs.append( - { - "artifact_namespace": namespace, - "error": repr(exc), - } - ) - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - raise - inner_frames.append(tuning_df) - inner_runs.append(run_summary) - completed_runs.append( + for future in as_completed(future_to_spec): + spec = future_to_spec[future] + namespace = str(spec["namespace"]) + running.discard(namespace) + try: + tuning_df, run_summary = future.result() + except Exception as exc: + failed_runs.append( { - "artifact_namespace": run_summary["artifact_namespace"], - "reused_existing": bool(run_summary.get("reused_existing", False)), + "artifact_namespace": namespace, + "error": repr(exc), } ) _write_phase1_progress( @@ -1498,19 +1623,53 @@ def main(argv: list[str] | None = None) -> int: failed=failed_runs, workers=phase1_workers, ) + raise + record_success(tuning_df, run_summary) + _write_phase1_progress( + path=progress_path, + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + total=len(specs), + completed=completed_runs, + running=sorted(running), + failed=failed_runs, + workers=phase1_workers, + ) - inner_df = pd.concat(inner_frames, ignore_index=True) - inner_df.to_parquet(output_paths["inner_search"], index=False) - aggregated = _aggregate_inner_search(inner_df) - aggregated.to_parquet(output_paths["inner_aggregate"], index=False) - shortlist = _dedupe_designs(aggregated, top_k_inner) - shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) - aggregate_path = str(output_paths["inner_aggregate"]) - inner_search_path = str(output_paths["inner_search"]) - inner_search_winner = ( - _normalize_design_row(aggregated.iloc[0].to_dict()) if not aggregated.empty else {} - ) + inner_df = pd.concat(inner_frames, ignore_index=True) + inner_df.to_parquet(output_paths["inner_search"], index=False) + aggregated = _aggregate_inner_search(inner_df) + aggregated.to_parquet(output_paths["inner_aggregate"], index=False) + shortlist = _dedupe_designs(aggregated, top_k_inner) + shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) + return Phase1Result( + shortlist=shortlist, + aggregated=aggregated, + inner_runs=inner_runs, + aggregate_path=str(output_paths["inner_aggregate"]), + inner_search_path=str(output_paths["inner_search"]), + inner_search_winner=_normalize_design_row(aggregated.iloc[0].to_dict()) + if not aggregated.empty + else {}, + ) + +def _run_phase1_oot_confirmation( + *, + run_tag: str, + env: dict[str, str], + shortlist: pd.DataFrame, + output_paths: dict[str, Path], + alpha_candidates_95: list[float], + partition_candidates: list[str], + partition_probability_sources: list[str], + n_score_bins_candidates: list[int], + fallback_modes: list[str], + score_scale_families: list[str], + calibration_fractions: list[float], + sidecar_cfg: dict[str, Any], + validation_cfg: dict[str, Any], +) -> Phase1ConfirmationResult: if shortlist.empty: raise RuntimeError("No phase1 shortlist candidates available for OOT confirmation.") @@ -1540,7 +1699,6 @@ def main(argv: list[str] | None = None) -> int: best_phase1 = next( candidate for candidate in phase1_candidates if candidate["namespace"] == best_phase1_ns ) - final_namespace = best_phase1["namespace"] final_policy = dict(best_phase1["policy_status"]) final_sets = dict(best_phase1["set_status"]) final_decision = ( @@ -1548,92 +1706,255 @@ def main(argv: list[str] | None = None) -> int: if _acceptance_pass(final_policy, validation_cfg) else "keep_current_canonical" ) + return Phase1ConfirmationResult( + candidates=phase1_candidates, + frame=phase1_df, + best_namespace=best_phase1_ns, + final_policy=final_policy, + final_sets=final_sets, + final_decision=final_decision, + final_namespace=str(best_phase1["namespace"]), + ) + + +def _phase2_run_reason(*, force_phase2: bool, phase2_always_evaluate: bool) -> str: + if force_phase2: + return "forced" + if phase2_always_evaluate: + return "always_evaluate" + return "phase1_acceptance_fail" + + +def _maybe_apply_phase2( + *, + run_tag: str, + upstream_run_tag: str, + env: dict[str, str], + aggregated: pd.DataFrame, + phase1: Phase1ConfirmationResult, + phase1_only: bool, + force_phase2: bool, + alpha_candidates_95: list[float], + tuning_holdout_ratios: list[float], + inner_random_states: list[int], + partition_candidates: list[str], + partition_probability_sources: list[str], + n_score_bins_candidates: list[int], + fallback_modes: list[str], + score_scale_families: list[str], + calibration_fractions: list[float], + phase2_cfg: dict[str, Any], + sidecar_cfg: dict[str, Any], + validation_cfg: dict[str, Any], +) -> PromotionResult: + final_policy = dict(phase1.final_policy) + final_sets = dict(phase1.final_sets) + final_decision = str(phase1.final_decision) + final_namespace = str(phase1.final_namespace) phase2_summary: dict[str, Any] | None = None - phase2_always_evaluate = bool(phase2_cfg.get("always_evaluate", False)) or bool( - args.force_phase2 - ) - phase2_run_reason = ( - "forced" - if bool(args.force_phase2) - else "always_evaluate" - if phase2_always_evaluate - else "phase1_acceptance_fail" - ) - phase1_policy = dict(final_policy) - phase1_sets = dict(final_sets) - phase1_decision = str(final_decision) - phase1_namespace = str(final_namespace) - if ( + phase2_always_evaluate = bool(phase2_cfg.get("always_evaluate", False)) or bool(force_phase2) + should_run_phase2 = ( (phase2_always_evaluate or (not _acceptance_pass(final_policy, validation_cfg))) - and (not bool(args.phase1_only)) + and (not phase1_only) and bool(phase2_cfg.get("enabled", True)) - ): - phase2_decision, phase2_policy, phase2_sets, phase2_summary = _run_phase2_search( + ) + if not should_run_phase2: + return PromotionResult( + final_policy=final_policy, + final_sets=final_sets, + final_decision=final_decision, + final_namespace=final_namespace, + phase2_summary=phase2_summary, + ) + + phase2_decision, phase2_policy, phase2_sets, phase2_summary = _run_phase2_search( + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + env=env, + aggregated=aggregated, + phase1_candidates_frame=phase1.frame, + alpha_candidates_95=alpha_candidates_95, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + calibration_fractions=calibration_fractions, + phase2_cfg=phase2_cfg, + sidecar_cfg=sidecar_cfg, + validation_cfg=validation_cfg, + ) + phase2_applied, phase2_apply_reason = _phase2_should_replace_phase1( + phase1_policy=phase1.final_policy, + phase2_policy=phase2_policy, + validation_cfg=validation_cfg, + ) + if phase2_summary is not None: + phase2_summary = { + **phase2_summary, + "run_reason": _phase2_run_reason( + force_phase2=force_phase2, + phase2_always_evaluate=phase2_always_evaluate, + ), + "always_evaluate": bool(phase2_always_evaluate), + "applied_to_final": bool(phase2_applied), + "apply_reason": phase2_apply_reason, + "phase1_namespace": phase1.final_namespace, + "phase1_decision": phase1.final_decision, + "phase2_decision": phase2_decision, + } + if phase2_applied: + final_decision = phase2_decision + final_policy = phase2_policy + final_sets = phase2_sets + if phase2_summary and phase2_summary.get("final_namespace"): + final_namespace = str(phase2_summary["final_namespace"]) + elif _acceptance_pass(phase1.final_policy, validation_cfg): + final_decision = phase1.final_decision + final_policy = dict(phase1.final_policy) + final_sets = dict(phase1.final_sets) + final_namespace = phase1.final_namespace + + return PromotionResult( + final_policy=final_policy, + final_sets=final_sets, + final_decision=final_decision, + final_namespace=final_namespace, + phase2_summary=phase2_summary, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + run_tag = str(args.run_tag).strip() + upstream_run_tag = str(args.upstream_canonical_run_tag).strip() + resume_from_run_tag = str(args.resume_from_run_tag).strip() if args.resume_from_run_tag else "" + mode = "derived_resume" if resume_from_run_tag else "fresh_search" + + profile = _profile_cfg(args.pipeline_profile) + cfg = _phase1_cfg(profile) + phase2_cfg = _phase2_cfg(profile) + sidecar_cfg = _sidecar_cfg(profile) + validation_cfg = _validation_cfg(profile) + output_paths = _reopen_artifact_paths(run_tag) + + env = os.environ.copy() + env["PIPELINE_RUN_TAG"] = run_tag + env["UPSTREAM_CANONICAL_RUN_TAG"] = upstream_run_tag + + alpha_candidates_90 = cfg.get("alpha_candidates_90", [0.09, 0.095, 0.10, 0.105, 0.11, 0.12]) + alpha_candidates_95 = cfg.get("alpha_candidates_95", [0.045, 0.05, 0.055, 0.06]) + partition_candidates = cfg.get( + "partition_candidates", + ["grade", "score_decile_mondrian", "grade_x_scoreband_mondrian"], + ) + partition_probability_sources = cfg.get("partition_probability_sources", ["calibrated", "raw"]) + n_score_bins_candidates = cfg.get("n_score_bins_candidates", [5, 10, 15, 20]) + min_group_sizes = cfg.get("min_group_sizes", [100, 150, 250, 500, 1000]) + fallback_modes = cfg.get("fallback_modes", ["grade_then_global", "global_only"]) + score_scale_families = cfg.get( + "score_scale_families", + ["none", "bernoulli_sqrt", "bernoulli_sqrt_clipped_0.02", "bernoulli_sqrt_clipped_0.05"], + ) + calibration_fractions = cfg.get("calibration_fractions", [0.25, 0.50, 0.75, 1.00]) + tuning_holdout_ratios = cfg.get("tuning_holdout_ratios", [0.20, 0.30]) + inner_random_states = cfg.get("inner_random_states", [42, 314, 2026]) + top_k_inner = int(validation_cfg.get("top_k_inner", 3)) + configured_workers = int(cfg.get("parallel_workers", 1) or 1) + phase1_workers = int(args.phase1_workers or configured_workers or 1) + phase1_workers = max(1, phase1_workers) + resume_completed_inner = not bool(args.no_resume_completed_inner) + + if resume_from_run_tag: + phase1_result = _phase1_from_resume( + resume_from_run_tag=resume_from_run_tag, + top_k_inner=top_k_inner, + output_paths=output_paths, + ) + else: + phase1_result = _run_phase1_inner_search( run_tag=run_tag, upstream_run_tag=upstream_run_tag, env=env, - aggregated=aggregated, - phase1_candidates_frame=phase1_df, + output_paths=output_paths, + phase1_workers=phase1_workers, + top_k_inner=top_k_inner, + alpha_candidates_90=alpha_candidates_90, alpha_candidates_95=alpha_candidates_95, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, partition_candidates=partition_candidates, partition_probability_sources=partition_probability_sources, n_score_bins_candidates=n_score_bins_candidates, + min_group_sizes=min_group_sizes, fallback_modes=fallback_modes, score_scale_families=score_scale_families, calibration_fractions=calibration_fractions, - phase2_cfg=phase2_cfg, - sidecar_cfg=sidecar_cfg, - validation_cfg=validation_cfg, - ) - phase2_applied, phase2_apply_reason = _phase2_should_replace_phase1( - phase1_policy=phase1_policy, - phase2_policy=phase2_policy, - validation_cfg=validation_cfg, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, + resume_completed_inner=resume_completed_inner, ) - if phase2_summary is not None: - phase2_summary = { - **phase2_summary, - "run_reason": phase2_run_reason, - "always_evaluate": bool(phase2_always_evaluate), - "applied_to_final": bool(phase2_applied), - "apply_reason": phase2_apply_reason, - "phase1_namespace": phase1_namespace, - "phase1_decision": phase1_decision, - "phase2_decision": phase2_decision, - } - if phase2_applied: - final_decision = phase2_decision - final_policy = phase2_policy - final_sets = phase2_sets - if phase2_summary and phase2_summary.get("final_namespace"): - final_namespace = str(phase2_summary["final_namespace"]) - elif _acceptance_pass(phase1_policy, validation_cfg): - final_decision = phase1_decision - final_policy = phase1_policy - final_sets = phase1_sets - final_namespace = phase1_namespace + shortlist = phase1_result.shortlist + aggregated = phase1_result.aggregated + + phase1_confirmation = _run_phase1_oot_confirmation( + run_tag=run_tag, + env=env, + shortlist=shortlist, + output_paths=output_paths, + alpha_candidates_95=alpha_candidates_95, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + calibration_fractions=calibration_fractions, + sidecar_cfg=sidecar_cfg, + validation_cfg=validation_cfg, + ) + final_result = _maybe_apply_phase2( + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + env=env, + aggregated=aggregated, + phase1=phase1_confirmation, + phase1_only=bool(args.phase1_only), + force_phase2=bool(args.force_phase2), + alpha_candidates_95=alpha_candidates_95, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + calibration_fractions=calibration_fractions, + phase2_cfg=phase2_cfg, + sidecar_cfg=sidecar_cfg, + validation_cfg=validation_cfg, + ) _write_consolidated_status( run_tag=run_tag, upstream_run_tag=upstream_run_tag, pipeline_profile=str(args.pipeline_profile), mode=mode, - inner_search_path=inner_search_path, - aggregate_path=aggregate_path, + inner_search_path=phase1_result.inner_search_path, + aggregate_path=phase1_result.aggregate_path, shortlist_path=str(output_paths["phase1_shortlist"]), phase1_final_path=str(output_paths["phase1_final_candidates"]), - inner_search_runs=inner_runs, - inner_search_winner=inner_search_winner, - best_phase1_namespace=best_phase1_ns, - final_policy=final_policy, - final_sets=final_sets, - final_decision=final_decision, - final_namespace=final_namespace, - phase2_summary=phase2_summary, - resume_meta=resume_meta, + inner_search_runs=phase1_result.inner_runs, + inner_search_winner=phase1_result.inner_search_winner, + best_phase1_namespace=phase1_confirmation.best_namespace, + final_policy=final_result.final_policy, + final_sets=final_result.final_sets, + final_decision=final_result.final_decision, + final_namespace=final_result.final_namespace, + phase2_summary=final_result.phase2_summary, + resume_meta=phase1_result.resume_meta, ) return 0 diff --git a/scripts/search/run_conformal_search.py b/scripts/search/run_conformal_search.py index 8a438ff..f131742 100644 --- a/scripts/search/run_conformal_search.py +++ b/scripts/search/run_conformal_search.py @@ -1,20 +1,24 @@ -"""Organized search entrypoint for conformal benchmarking/tuning.""" +"""Retired conformal search entrypoint. + +The former generic ``scripts.run_long_pipeline`` orchestrator was removed when +the IJDS paper lane was narrowed around frozen artifacts. Keep this file as a +readable stop sign for old commands instead of failing with an import error. +""" from __future__ import annotations import sys -from scripts.run_long_pipeline import main as _main - def main(argv: list[str] | None = None) -> int: - return _main( - argv, - default_pipeline_family="search_conformal", - default_sampling_profile="champion64safe", - default_include_rapids=False, - default_include_notebooks=False, + _ = argv + sys.stderr.write( + "scripts/search/run_conformal_search.py is retired. Use the frozen " + "IJDS artifacts plus paper evidence stages, or start a new isolated " + "experiment under scripts/search/run_conformal_reopen_search.py with " + "an explicit run tag and drift plan.\n" ) + return 2 if __name__ == "__main__": diff --git a/scripts/search/run_pool93_ijds_local_refinement.py b/scripts/search/run_pool93_ijds_local_refinement.py index 4d0e3e2..eee8edb 100644 --- a/scripts/search/run_pool93_ijds_local_refinement.py +++ b/scripts/search/run_pool93_ijds_local_refinement.py @@ -19,6 +19,7 @@ import sys import time from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait +from dataclasses import dataclass from datetime import UTC, datetime from itertools import product from pathlib import Path @@ -46,7 +47,15 @@ _compute_intervals_at_alpha, _load_aligned_dataset, ) -from src.optimization.portfolio_model import optimize_portfolio_allocation # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + IJDS_DECLARED_ALPHA_GRID, + add_policy_aware_bound_columns, + compute_funded_certificate_metrics, +) +from src.optimization.portfolio_model import ( # noqa: E402 + optimize_portfolio_allocation, + solution_allocation_vector, +) from src.utils.pipeline_runtime import ( # noqa: E402 atomic_write_json, atomic_write_parquet, @@ -56,7 +65,57 @@ STAGE_NAME = "pool93_ijds_local_refinement" DECLARED_RETURN_FLOOR = 170464.54 -DEFAULT_ALPHA_GRID = [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] +DEFAULT_ALPHA_GRID = list(IJDS_DECLARED_ALPHA_GRID) +VALID_PROFILES = { + "stage1", + "expanded", + "claim_expanded", + "claim_micro", + "claim_micro_ext", + "claim_bound_closure", + "claim_bound_floor_closure", + "claim_bound_terminal", +} +ANCHOR_REASONS = ( + (96, "source_exact_max_return"), + (219, "source_low_gamma_cp_return_floor"), + (223, "source_low_weighted_miscoverage_high_return"), +) +CLAIM_ROW_FIELDS = ( + "claim_rank", + "local_candidate_id", + "local_family", + "anchor_rank", + "source_reason", + "risk_tolerance", + "policy_mode", + "gamma", + "delta_cap_quantile", + "tail_focus_quantile", + "uncertainty_aversion", + "alpha01_realized_total_return", + "return_floor_surplus", + "alpha01_gamma_cp", + "alpha01_gamma_internalized", + "alpha01_gamma_residual", + "alpha01_weighted_miscoverage_V", + "alpha01_endpoint_budget", + "alpha01_endpoint_budget_upper", + "alpha01_markov_loss_threshold", + "alpha01_markov_loss_cap", + "alpha01_weighted_pd_true", + "alpha01_empirical_coverage_funded", + "alpha_exact_pass_count", + "alpha_exact_check_count", + "alpha_mean_gamma_cp", + "alpha_mean_weighted_miscoverage_V", + "return_score", + "bound_score", + "v_score", + "ijds_balanced_score", + "n_funded_mean", + "allocator_backends", +) DEFAULT_SOURCE_BOUND_EVAL = ( ROOT / "data/processed/experiments/champion_reopen/" "champion-reopen-2026-06-19__hpo-wave1__pool93__portfolio-stage1-fast1-claim-26-06/" @@ -68,6 +127,20 @@ "portfolio/portfolio_bound_aware_selection_highspy.json" ) + +@dataclass(frozen=True) +class Pool93Paths: + output_dir: Path + model_dir: Path + checkpoint_dir: Path + status_path: Path + candidates_path: Path + bound_eval_path: Path + leaderboard_path: Path + claim_summary_path: Path + manifest_path: Path + + _WORKER_ALIGNED: pd.DataFrame | None = None @@ -185,49 +258,40 @@ def _add_candidate( seen.add(key) -def _generate_candidate_grid( - anchors: pd.DataFrame, +def _candidate_frame(rows: list[dict[str, Any]]) -> pd.DataFrame: + candidates = pd.DataFrame(rows).reset_index(drop=True) + candidates.insert(0, "local_candidate_id", np.arange(1, len(candidates) + 1, dtype=int)) + return candidates + + +def _anchor_policy( + anchor_by_rank: dict[int, dict[str, Any]], + rank: int, *, - profile: str, solver_backend: str, -) -> pd.DataFrame: - profile = str(profile).strip().lower() - valid_profiles = { - "stage1", - "expanded", - "claim_expanded", - "claim_micro", - "claim_micro_ext", - "claim_bound_closure", - "claim_bound_floor_closure", - "claim_bound_terminal", - } - if profile not in valid_profiles: - raise ValueError(f"profile must be one of {sorted(valid_profiles)}") - - anchor_by_rank = {int(row.candidate_rank): row for row in anchors.itertuples(index=False)} - rows: list[dict[str, Any]] = [] - seen: set[str] = set() +) -> dict[str, Any]: + row = anchor_by_rank[rank] + return _policy_base( + risk_tolerance=float(row["risk_tolerance"]), + policy_mode=str(row["policy_mode"]), + gamma=float(row["gamma"]), + uncertainty_aversion=float(row["uncertainty_aversion"]), + delta_cap_quantile=float(row["delta_cap_quantile"]), + tail_focus_quantile=float(row["tail_focus_quantile"]), + min_budget_utilization=float(row["min_budget_utilization"]), + pd_cap_slack_penalty=float(row["pd_cap_slack_penalty"]), + solver_backend=solver_backend, + ) - def anchor_policy(rank: int) -> dict[str, Any]: - row = anchor_by_rank[rank] - return _policy_base( - risk_tolerance=float(row.risk_tolerance), - policy_mode=str(row.policy_mode), - gamma=float(row.gamma), - uncertainty_aversion=float(row.uncertainty_aversion), - delta_cap_quantile=float(row.delta_cap_quantile), - tail_focus_quantile=float(row.tail_focus_quantile), - min_budget_utilization=float(row.min_budget_utilization), - pd_cap_slack_penalty=float(row.pd_cap_slack_penalty), - solver_backend=solver_backend, - ) - for rank, reason in [ - (96, "source_exact_max_return"), - (219, "source_low_gamma_cp_return_floor"), - (223, "source_low_weighted_miscoverage_high_return"), - ]: +def _add_anchor_candidates( + rows: list[dict[str, Any]], + seen: set[str], + *, + anchor_by_rank: dict[int, dict[str, Any]], + solver_backend: str, +) -> None: + for rank, reason in ANCHOR_REASONS: if rank in anchor_by_rank: _add_candidate( rows, @@ -235,682 +299,463 @@ def anchor_policy(rank: int) -> dict[str, Any]: family="anchor_policy", anchor_rank=rank, source_reason=reason, - policy=anchor_policy(rank), + policy=_anchor_policy(anchor_by_rank, rank, solver_backend=solver_backend), ) - if profile == "claim_micro": - # Final IJDS micro-refinement around the completed expanded frontier: - # - candidate 1206: tightest Markov cap above the declared return floor, - # - candidate 1665/1667: body/default low-V return-bound point, - # - candidate 1922: highest return under Markov cap <= 0.36, - # - candidate 2777/2857: economic endpoint. - # The grid is intentionally local and paper-facing, not a new generic - # champion search. - body_risks = _round_grid( - [0.1715 + 0.00025 * idx for idx in range(5)], - lo=0.14, - hi=0.24, - ) - body_gammas = _round_grid( - [0.545 + 0.005 * idx for idx in range(7)], - lo=0.0, - hi=1.0, - ) - body_aversions = [0.0, 0.0125, 0.025, 0.0375, 0.05, 0.0625, 0.075, 0.10] - for risk, gamma, aversion, mode in product( - body_risks, - body_gammas, - body_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_body_low_v", - anchor_rank=219, - source_reason="candidate1665_1667_body_default_micro", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - tight_risks = _round_grid( - [0.1700 + 0.00025 * idx for idx in range(5)], - lo=0.14, - hi=0.24, - ) - tight_gammas = _round_grid( - [0.575 + 0.005 * idx for idx in range(6)], - lo=0.0, - hi=1.0, - ) - tight_aversions = [0.15, 0.1625, 0.175, 0.1875, 0.20, 0.225] - for risk, gamma, aversion, mode in product( - tight_risks, - tight_gammas, - tight_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_bound_tight", - anchor_rank=219, - source_reason="candidate1206_tight_cap_micro", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) +def _capped_delta_values(mode: str, capped_values: tuple[float, ...]) -> tuple[float, ...]: + return (1.0,) if mode == "blended_uncertainty" else capped_values - high_return_risks = _round_grid( - [0.1725 + 0.00025 * idx for idx in range(7)], - lo=0.14, - hi=0.24, - ) - high_return_gammas = _round_grid( - [0.500 + 0.005 * idx for idx in range(6)], - lo=0.0, - hi=1.0, - ) - high_return_aversions = [0.05, 0.0625, 0.075, 0.0875, 0.10] - for risk, gamma, aversion, mode in product( - high_return_risks, - high_return_gammas, - high_return_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_high_return_cap036", - anchor_rank=219, - source_reason="candidate1922_return_cap036_micro", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - econ_risks = _round_grid( - [0.1565 + 0.00025 * idx for idx in range(9)], - lo=0.12, - hi=0.22, - ) - econ_gammas = _round_grid( - [0.44 + 0.005 * idx for idx in range(13)], - lo=0.0, - hi=1.0, - ) - econ_aversions = [0.1125, 0.125, 0.1375, 0.15] - for risk, gamma, aversion, tail_focus in product( - econ_risks, - econ_gammas, - econ_aversions, - [0.95, 1.0], - ): +def _add_blended_grid( + rows: list[dict[str, Any]], + seen: set[str], + *, + family: str, + anchor_rank: int, + source_reason: str, + risks: list[float], + gammas: list[float], + aversions: list[float], + solver_backend: str, + capped_delta_values: tuple[float, ...] = (0.95, 1.0), +) -> None: + for risk, gamma, aversion, mode in product( + risks, + gammas, + aversions, + ["blended_uncertainty", "capped_blended_uncertainty"], + ): + for delta_cap in _capped_delta_values(str(mode), capped_delta_values): _add_candidate( rows, seen, - family="claim_micro_economic_endpoint", - anchor_rank=96, - source_reason="candidate2777_2857_economic_endpoint_micro", + family=family, + anchor_rank=anchor_rank, + source_reason=source_reason, policy=_policy_base( risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", + policy_mode=str(mode), gamma=gamma, uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, + delta_cap_quantile=delta_cap, solver_backend=solver_backend, ), ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates - if profile == "claim_micro_ext": - # Surgical extensions from the completed claim_micro frontier. These - # neighborhoods target only exposed claim boundaries, not a fresh broad - # portfolio search: - # - body/cap<=0.345 polish around candidates 37/205, - # - bound-tight endpoint around candidate 949, - # - cap<=0.36 return endpoint around candidate 1975, - # - economic endpoint around candidate 2122. - body_risks = _round_grid( - [0.17125 + 0.000125 * idx for idx in range(11)], - lo=0.14, - hi=0.24, - ) - body_gammas = _round_grid( - [0.5475, 0.55, 0.5525, 0.555, 0.5575], - lo=0.0, - hi=1.0, +def _add_tail_grid( + rows: list[dict[str, Any]], + seen: set[str], + *, + family: str, + anchor_rank: int, + source_reason: str, + risks: list[float], + gammas: list[float], + aversions: list[float], + tail_focus_values: list[float], + solver_backend: str, +) -> None: + for risk, gamma, aversion, tail_focus in product( + risks, + gammas, + aversions, + tail_focus_values, + ): + _add_candidate( + rows, + seen, + family=family, + anchor_rank=anchor_rank, + source_reason=source_reason, + policy=_policy_base( + risk_tolerance=risk, + policy_mode="tail_blended_uncertainty", + gamma=gamma, + uncertainty_aversion=aversion, + tail_focus_quantile=tail_focus, + solver_backend=solver_backend, + ), ) - body_aversions = [0.025, 0.0375, 0.05, 0.0625] - for risk, gamma, aversion, mode in product( - body_risks, - body_gammas, - body_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.975, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_ext_body_cap345", - anchor_rank=219, - source_reason="candidate37_205_body_cap345_extension", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - tight_risks = _round_grid( - [0.1690 + 0.00025 * idx for idx in range(8)], - lo=0.14, - hi=0.24, - ) - tight_gammas = _round_grid( - [0.600 + 0.005 * idx for idx in range(11)], - lo=0.0, - hi=1.0, - ) - tight_aversions = [0.2125, 0.225, 0.2375, 0.25, 0.2625, 0.275] - for risk, gamma, aversion, mode in product( - tight_risks, - tight_gammas, - tight_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_ext_bound_tight", - anchor_rank=219, - source_reason="candidate949_bound_tight_extension", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - cap036_risks = _round_grid( - [0.17375 + 0.00025 * idx for idx in range(10)], - lo=0.14, - hi=0.24, - ) - cap036_gammas = _round_grid( - [0.505 + 0.0025 * idx for idx in range(9)], - lo=0.0, - hi=1.0, - ) - cap036_aversions = [0.0625, 0.075, 0.0875, 0.10] - for risk, gamma, aversion, mode in product( - cap036_risks, - cap036_gammas, - cap036_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_ext_cap036_return", - anchor_rank=219, - source_reason="candidate1975_cap036_return_extension", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) +def _append_claim_micro_candidates( + rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str +) -> None: + _add_blended_grid( + rows, + seen, + family="claim_micro_body_low_v", + anchor_rank=219, + source_reason="candidate1665_1667_body_default_micro", + risks=_round_grid([0.1715 + 0.00025 * idx for idx in range(5)], lo=0.14, hi=0.24), + gammas=_round_grid([0.545 + 0.005 * idx for idx in range(7)], lo=0.0, hi=1.0), + aversions=[0.0, 0.0125, 0.025, 0.0375, 0.05, 0.0625, 0.075, 0.10], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_micro_bound_tight", + anchor_rank=219, + source_reason="candidate1206_tight_cap_micro", + risks=_round_grid([0.1700 + 0.00025 * idx for idx in range(5)], lo=0.14, hi=0.24), + gammas=_round_grid([0.575 + 0.005 * idx for idx in range(6)], lo=0.0, hi=1.0), + aversions=[0.15, 0.1625, 0.175, 0.1875, 0.20, 0.225], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_micro_high_return_cap036", + anchor_rank=219, + source_reason="candidate1922_return_cap036_micro", + risks=_round_grid([0.1725 + 0.00025 * idx for idx in range(7)], lo=0.14, hi=0.24), + gammas=_round_grid([0.500 + 0.005 * idx for idx in range(6)], lo=0.0, hi=1.0), + aversions=[0.05, 0.0625, 0.075, 0.0875, 0.10], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_tail_grid( + rows, + seen, + family="claim_micro_economic_endpoint", + anchor_rank=96, + source_reason="candidate2777_2857_economic_endpoint_micro", + risks=_round_grid([0.1565 + 0.00025 * idx for idx in range(9)], lo=0.12, hi=0.22), + gammas=_round_grid([0.44 + 0.005 * idx for idx in range(13)], lo=0.0, hi=1.0), + aversions=[0.1125, 0.125, 0.1375, 0.15], + tail_focus_values=[0.95, 1.0], + solver_backend=solver_backend, + ) - econ_risks = _round_grid( - [0.15625 + 0.000125 * idx for idx in range(9)], - lo=0.12, - hi=0.22, - ) - econ_gammas = _round_grid( - [0.400 + 0.005 * idx for idx in range(10)], - lo=0.0, - hi=1.0, - ) - econ_aversions = [0.125, 0.1375, 0.15] - for risk, gamma, aversion, tail_focus in product( - econ_risks, - econ_gammas, - econ_aversions, - [0.90, 0.925, 0.95, 1.0], - ): - _add_candidate( - rows, - seen, - family="claim_micro_ext_economic_endpoint", - anchor_rank=96, - source_reason="candidate2122_economic_endpoint_extension", - policy=_policy_base( - risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", - gamma=gamma, - uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, - solver_backend=solver_backend, - ), - ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates +def _append_claim_micro_ext_candidates( + rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str +) -> None: + _add_blended_grid( + rows, + seen, + family="claim_micro_ext_body_cap345", + anchor_rank=219, + source_reason="candidate37_205_body_cap345_extension", + risks=_round_grid([0.17125 + 0.000125 * idx for idx in range(11)], lo=0.14, hi=0.24), + gammas=_round_grid([0.5475, 0.55, 0.5525, 0.555, 0.5575], lo=0.0, hi=1.0), + aversions=[0.025, 0.0375, 0.05, 0.0625], + capped_delta_values=(0.975, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_micro_ext_bound_tight", + anchor_rank=219, + source_reason="candidate949_bound_tight_extension", + risks=_round_grid([0.1690 + 0.00025 * idx for idx in range(8)], lo=0.14, hi=0.24), + gammas=_round_grid([0.600 + 0.005 * idx for idx in range(11)], lo=0.0, hi=1.0), + aversions=[0.2125, 0.225, 0.2375, 0.25, 0.2625, 0.275], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_micro_ext_cap036_return", + anchor_rank=219, + source_reason="candidate1975_cap036_return_extension", + risks=_round_grid([0.17375 + 0.00025 * idx for idx in range(10)], lo=0.14, hi=0.24), + gammas=_round_grid([0.505 + 0.0025 * idx for idx in range(9)], lo=0.0, hi=1.0), + aversions=[0.0625, 0.075, 0.0875, 0.10], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_tail_grid( + rows, + seen, + family="claim_micro_ext_economic_endpoint", + anchor_rank=96, + source_reason="candidate2122_economic_endpoint_extension", + risks=_round_grid([0.15625 + 0.000125 * idx for idx in range(9)], lo=0.12, hi=0.22), + gammas=_round_grid([0.400 + 0.005 * idx for idx in range(10)], lo=0.0, hi=1.0), + aversions=[0.125, 0.1375, 0.15], + tail_focus_values=[0.90, 0.925, 0.95, 1.0], + solver_backend=solver_backend, + ) + +def _append_bound_closure_candidates( + rows: list[dict[str, Any]], + seen: set[str], + *, + profile: str, + solver_backend: str, +) -> None: if profile == "claim_bound_closure": - # Final IJDS bound-endpoint closure. The completed claim_micro_ext run - # found a monotone-looking cap reduction up to gamma=0.65, while - # uncertainty_aversion was already mostly saturated. This tiny extension - # tests only whether the lower-bound endpoint can move; it is not a new - # body-policy or economic-endpoint search. - closure_risks = _round_grid( - [0.1685 + 0.00025 * idx for idx in range(10)], - lo=0.14, - hi=0.24, - ) - closure_gammas = _round_grid( - [0.65 + 0.01 * idx for idx in range(11)], - lo=0.0, - hi=1.0, + _add_blended_grid( + rows, + seen, + family="claim_bound_closure_low_cap", + anchor_rank=219, + source_reason="micro_ext_min_markov_cap_endpoint_closure", + risks=_round_grid([0.1685 + 0.00025 * idx for idx in range(10)], lo=0.14, hi=0.24), + gammas=_round_grid([0.65 + 0.01 * idx for idx in range(11)], lo=0.0, hi=1.0), + aversions=[0.25, 0.275, 0.30, 0.325, 0.35], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, ) - closure_aversions = [0.25, 0.275, 0.30, 0.325, 0.35] - for risk, gamma, aversion, mode in product( - closure_risks, - closure_gammas, - closure_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_bound_closure_low_cap", - anchor_rank=219, - source_reason="micro_ext_min_markov_cap_endpoint_closure", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates - + return if profile == "claim_bound_floor_closure": - # Last bounded IJDS endpoint check: the completed bound closure lowered - # the Markov cap to 0.298369 at the grid boundary (low tau, high gamma, - # high aversion) while still preserving a positive return-floor surplus. - # This profile tests whether the appendix/theory endpoint can cross the - # cleaner cap<0.29 threshold; it should not be used to replace the paper - # body/default policy. - floor_risks = _round_grid( - [0.16775 + 0.000125 * idx for idx in range(13)], - lo=0.14, - hi=0.24, - ) - floor_gammas = _round_grid( - [0.75 + 0.01 * idx for idx in range(10)], - lo=0.0, - hi=1.0, + _add_blended_grid( + rows, + seen, + family="claim_bound_floor_closure_low_cap", + anchor_rank=219, + source_reason="bound_closure_cap029_floor_threshold", + risks=_round_grid([0.16775 + 0.000125 * idx for idx in range(13)], lo=0.14, hi=0.24), + gammas=_round_grid([0.75 + 0.01 * idx for idx in range(10)], lo=0.0, hi=1.0), + aversions=[0.325, 0.35, 0.375, 0.40, 0.425, 0.45], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, ) - floor_aversions = [0.325, 0.35, 0.375, 0.40, 0.425, 0.45] - for risk, gamma, aversion, mode in product( - floor_risks, - floor_gammas, - floor_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_bound_floor_closure_low_cap", - anchor_rank=219, - source_reason="bound_closure_cap029_floor_threshold", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates - if profile == "claim_bound_terminal": - # Terminal IJDS endpoint search. This is intentionally wider than the - # prior closures, but still only targets the final bound-tight endpoint: - # cap<0.285/0.280/0.275 if feasible, with positive return-floor surplus. - # It should not be used to replace the body/default or economic endpoint. - ultra_risks = _round_grid( - [0.16675 + 0.000125 * idx for idx in range(29)], - lo=0.14, - hi=0.24, - ) - ultra_gammas = _round_grid( - [0.84 + 0.005 * idx for idx in range(31)], - lo=0.0, - hi=1.0, - ) - ultra_aversions = [0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60, 0.65, 0.70] - for risk, gamma, aversion, mode in product( - ultra_risks, - ultra_gammas, - ultra_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_bound_terminal_ultra_low_cap", - anchor_rank=219, - source_reason="terminal_cap_threshold_search", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - recovery_risks = _round_grid( - [0.1680 + 0.000125 * idx for idx in range(29)], - lo=0.14, - hi=0.24, - ) - recovery_gammas = _round_grid( - [0.80 + 0.005 * idx for idx in range(25)], - lo=0.0, - hi=1.0, - ) - recovery_aversions = [0.35, 0.375, 0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60] - for risk, gamma, aversion, mode in product( - recovery_risks, - recovery_gammas, - recovery_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_bound_terminal_return_recovery", - anchor_rank=219, - source_reason="terminal_best_return_under_low_cap", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) +def _append_bound_terminal_candidates( + rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str +) -> None: + _add_blended_grid( + rows, + seen, + family="claim_bound_terminal_ultra_low_cap", + anchor_rank=219, + source_reason="terminal_cap_threshold_search", + risks=_round_grid([0.16675 + 0.000125 * idx for idx in range(29)], lo=0.14, hi=0.24), + gammas=_round_grid([0.84 + 0.005 * idx for idx in range(31)], lo=0.0, hi=1.0), + aversions=[0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60, 0.65, 0.70], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_bound_terminal_return_recovery", + anchor_rank=219, + source_reason="terminal_best_return_under_low_cap", + risks=_round_grid([0.1680 + 0.000125 * idx for idx in range(29)], lo=0.14, hi=0.24), + gammas=_round_grid([0.80 + 0.005 * idx for idx in range(25)], lo=0.0, hi=1.0), + aversions=[0.35, 0.375, 0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates - if 96 in anchor_by_rank: - base = anchor_by_rank[96] - risk_offsets = [-0.004, -0.002, -0.001, 0.0, 0.001, 0.002, 0.004] - gamma_offsets = [-0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05] - aversions = [0.05, 0.075, 0.10, 0.125, 0.15] - if profile == "expanded": - risk_offsets = [ - -0.006, - -0.004, - -0.003, - -0.002, - -0.001, - 0.0, - 0.001, - 0.002, - 0.003, - 0.004, - 0.006, - ] - gamma_offsets = [ - -0.075, - -0.05, - -0.035, - -0.025, - -0.01, - 0.0, - 0.01, - 0.025, - 0.035, - 0.05, - 0.075, - ] - aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] - risks = _round_grid( - [float(base.risk_tolerance) + x for x in risk_offsets], lo=0.12, hi=0.22 +def _append_max_return_neighborhood( + rows: list[dict[str, Any]], + seen: set[str], + *, + anchor_by_rank: dict[int, dict[str, Any]], + profile: str, + solver_backend: str, +) -> None: + if 96 not in anchor_by_rank: + return + base = anchor_by_rank[96] + risk_offsets = [-0.004, -0.002, -0.001, 0.0, 0.001, 0.002, 0.004] + gamma_offsets = [-0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05] + aversions = [0.05, 0.075, 0.10, 0.125, 0.15] + if profile == "expanded": + risk_offsets = [ + -0.006, + -0.004, + -0.003, + -0.002, + -0.001, + 0.0, + 0.001, + 0.002, + 0.003, + 0.004, + 0.006, + ] + gamma_offsets = [-0.075, -0.05, -0.035, -0.025, -0.01, 0.0, 0.01, 0.025, 0.035, 0.05, 0.075] + aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] + risks = _round_grid([float(base["risk_tolerance"]) + x for x in risk_offsets], lo=0.12, hi=0.22) + gammas = _round_grid([float(base["gamma"]) + x for x in gamma_offsets], lo=0.0, hi=1.0) + + for risk, gamma, aversion in product(risks, gammas, aversions): + _add_candidate( + rows, + seen, + family="max_return_segment_relative_local", + anchor_rank=96, + source_reason="rank96_local_dense", + policy=_policy_base( + risk_tolerance=risk, + policy_mode="segment_relative_tail_blended_uncertainty", + gamma=gamma, + uncertainty_aversion=aversion, + solver_backend=solver_backend, + ), ) - gammas = _round_grid([float(base.gamma) + x for x in gamma_offsets], lo=0.0, hi=1.0) - for risk, gamma, aversion in product(risks, gammas, aversions): - _add_candidate( - rows, - seen, - family="max_return_segment_relative_local", - anchor_rank=96, - source_reason="rank96_local_dense", - policy=_policy_base( - risk_tolerance=risk, - policy_mode="segment_relative_tail_blended_uncertainty", - gamma=gamma, - uncertainty_aversion=aversion, - solver_backend=solver_backend, - ), - ) + _add_tail_grid( + rows, + seen, + family="max_return_tail_local", + anchor_rank=96, + source_reason="rank96_tail_sensitivity", + risks=risks[1:-1] if len(risks) > 2 else risks, + gammas=gammas[1:-1] if len(gammas) > 2 else gammas, + aversions=[0.075, 0.10, 0.125] if profile == "stage1" else aversions, + tail_focus_values=[0.95, 1.0] if profile == "stage1" else [0.90, 0.95, 1.0], + solver_backend=solver_backend, + ) - tail_risks = risks[1:-1] if len(risks) > 2 else risks - tail_gammas = gammas[1:-1] if len(gammas) > 2 else gammas - tail_focus_values = [0.95, 1.0] if profile == "stage1" else [0.90, 0.95, 1.0] - tail_aversions = [0.075, 0.10, 0.125] if profile == "stage1" else aversions - for risk, gamma, aversion, tail_focus in product( - tail_risks, tail_gammas, tail_aversions, tail_focus_values - ): - _add_candidate( - rows, - seen, - family="max_return_tail_local", - anchor_rank=96, - source_reason="rank96_tail_sensitivity", - policy=_policy_base( - risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", - gamma=gamma, - uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, - solver_backend=solver_backend, - ), - ) - bound_risk_centers = [] - bound_gamma_centers = [] +def _append_bound_neighborhood( + rows: list[dict[str, Any]], + seen: set[str], + *, + anchor_by_rank: dict[int, dict[str, Any]], + profile: str, + solver_backend: str, +) -> None: + risk_centers = [] + gamma_centers = [] for rank in (219, 223): if rank in anchor_by_rank: row = anchor_by_rank[rank] - bound_risk_centers.append(float(row.risk_tolerance)) - bound_gamma_centers.append(float(row.gamma)) - if bound_risk_centers: - risk_offsets = [-0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005] - gamma_offsets = [-0.05, -0.025, 0.0, 0.025, 0.05] - aversions = [0.05, 0.075, 0.10, 0.125, 0.15] - if profile == "expanded": - risk_offsets = [-0.01, -0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005, 0.0075, 0.01] - gamma_offsets = [-0.075, -0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05, 0.075] - aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] - risks = _round_grid( - [center + offset for center in bound_risk_centers for offset in risk_offsets], - lo=0.14, - hi=0.24, - ) - gammas = _round_grid( - [center + offset for center in bound_gamma_centers for offset in gamma_offsets], - lo=0.0, - hi=1.0, - ) - for risk, gamma, aversion, mode in product( - risks, - gammas, - aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] - if mode == "capped_blended_uncertainty" and profile == "expanded": - delta_values = [0.90, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="bound_efficient_local", - anchor_rank=219 if abs(gamma - 0.45) <= abs(gamma - 0.40) else 223, - source_reason="rank219_rank223_bound_frontier", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - if profile == "claim_expanded": - # Densify only the claim-bearing neighborhoods discovered by stage1: - # the low-cap return-bound ridge around local candidates 462/466 and - # the economic frontier endpoint around local candidate 264. - bound_risks = _round_grid( - [0.1705 + 0.0005 * idx for idx in range(10)] + [0.1750], - lo=0.14, - hi=0.24, - ) - bound_gammas = _round_grid( - [0.49, 0.50, 0.51, 0.52, 0.535, 0.55, 0.575], - lo=0.0, - hi=1.0, - ) - bound_aversions = [0.05, 0.075, 0.10, 0.1125, 0.125, 0.1375, 0.15, 0.175] - for risk, gamma, aversion, mode in product( - bound_risks, - bound_gammas, - bound_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="bound_claim_refined_local", - anchor_rank=219, - source_reason="candidate462_466_return_bound_ridge", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - return_risks = _round_grid( - [0.1560 + 0.0005 * idx for idx in range(9)], - lo=0.12, - hi=0.22, - ) - return_gammas = _round_grid( - [0.44, 0.45, 0.46, 0.47, 0.475, 0.485, 0.495], - lo=0.0, - hi=1.0, - ) - return_aversions = [0.10, 0.1125, 0.125, 0.1375, 0.15] - for risk, gamma, aversion, tail_focus in product( - return_risks, - return_gammas, - return_aversions, - [0.95, 1.0], - ): + risk_centers.append(float(row["risk_tolerance"])) + gamma_centers.append(float(row["gamma"])) + if not risk_centers: + return + risk_offsets = [-0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005] + gamma_offsets = [-0.05, -0.025, 0.0, 0.025, 0.05] + aversions = [0.05, 0.075, 0.10, 0.125, 0.15] + capped_delta_values: tuple[float, ...] = (1.0,) + if profile == "expanded": + risk_offsets = [-0.01, -0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005, 0.0075, 0.01] + gamma_offsets = [-0.075, -0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05, 0.075] + aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] + capped_delta_values = (0.90, 1.0) + risks = _round_grid( + [center + offset for center in risk_centers for offset in risk_offsets], lo=0.14, hi=0.24 + ) + gammas = _round_grid( + [center + offset for center in gamma_centers for offset in gamma_offsets], lo=0.0, hi=1.0 + ) + for risk, gamma, aversion, mode in product( + risks, + gammas, + aversions, + ["blended_uncertainty", "capped_blended_uncertainty"], + ): + for delta_cap in _capped_delta_values(str(mode), capped_delta_values): _add_candidate( rows, seen, - family="max_return_claim_refined_local", - anchor_rank=96, - source_reason="candidate264_economic_frontier_endpoint", + family="bound_efficient_local", + anchor_rank=219 if abs(gamma - 0.45) <= abs(gamma - 0.40) else 223, + source_reason="rank219_rank223_bound_frontier", policy=_policy_base( risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", + policy_mode=str(mode), gamma=gamma, uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, + delta_cap_quantile=delta_cap, solver_backend=solver_backend, ), ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates + +def _append_claim_expanded_candidates( + rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str +) -> None: + _add_blended_grid( + rows, + seen, + family="bound_claim_refined_local", + anchor_rank=219, + source_reason="candidate462_466_return_bound_ridge", + risks=_round_grid( + [0.1705 + 0.0005 * idx for idx in range(10)] + [0.1750], lo=0.14, hi=0.24 + ), + gammas=_round_grid([0.49, 0.50, 0.51, 0.52, 0.535, 0.55, 0.575], lo=0.0, hi=1.0), + aversions=[0.05, 0.075, 0.10, 0.1125, 0.125, 0.1375, 0.15, 0.175], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_tail_grid( + rows, + seen, + family="max_return_claim_refined_local", + anchor_rank=96, + source_reason="candidate264_economic_frontier_endpoint", + risks=_round_grid([0.1560 + 0.0005 * idx for idx in range(9)], lo=0.12, hi=0.22), + gammas=_round_grid([0.44, 0.45, 0.46, 0.47, 0.475, 0.485, 0.495], lo=0.0, hi=1.0), + aversions=[0.10, 0.1125, 0.125, 0.1375, 0.15], + tail_focus_values=[0.95, 1.0], + solver_backend=solver_backend, + ) + + +def _generate_candidate_grid( + anchors: pd.DataFrame, + *, + profile: str, + solver_backend: str, +) -> pd.DataFrame: + profile = str(profile).strip().lower() + if profile not in VALID_PROFILES: + raise ValueError(f"profile must be one of {sorted(VALID_PROFILES)}") + + anchor_by_rank = {int(row["candidate_rank"]): row for row in anchors.to_dict(orient="records")} + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + _add_anchor_candidates( + rows, + seen, + anchor_by_rank=anchor_by_rank, + solver_backend=solver_backend, + ) + + if profile == "claim_micro": + _append_claim_micro_candidates(rows, seen, solver_backend=solver_backend) + return _candidate_frame(rows) + if profile == "claim_micro_ext": + _append_claim_micro_ext_candidates(rows, seen, solver_backend=solver_backend) + return _candidate_frame(rows) + if profile in {"claim_bound_closure", "claim_bound_floor_closure"}: + _append_bound_closure_candidates( + rows, + seen, + profile=profile, + solver_backend=solver_backend, + ) + return _candidate_frame(rows) + if profile == "claim_bound_terminal": + _append_bound_terminal_candidates(rows, seen, solver_backend=solver_backend) + return _candidate_frame(rows) + + _append_max_return_neighborhood( + rows, + seen, + anchor_by_rank=anchor_by_rank, + profile=profile, + solver_backend=solver_backend, + ) + _append_bound_neighborhood( + rows, + seen, + anchor_by_rank=anchor_by_rank, + profile=profile, + solver_backend=solver_backend, + ) + if profile == "claim_expanded": + _append_claim_expanded_candidates(rows, seen, solver_backend=solver_backend) + return _candidate_frame(rows) def _exact_policy_alpha( @@ -964,21 +809,19 @@ def _exact_policy_alpha( threads=max(1, int(threads)), solver_backend=str(policy["solver_backend"]), ) - if "allocation_vector" in solution: - alloc = np.asarray(solution["allocation_vector"], dtype=float) - else: - alloc = np.array( - [float(solution["allocation"].get(i, 0.0)) for i in range(len(aligned))], - dtype=float, - ) + alloc = solution_allocation_vector(solution, len(aligned)) total_allocated = float(np.sum(alloc * loan_amounts)) weights = (alloc * loan_amounts) / max(total_allocated, 1e-6) - funded_mask = weights > 1e-8 - miscoverage = (y_true > pd_high).astype(float) - weighted_miscoverage_v = float(np.sum(weights * miscoverage)) - weighted_pd_true = float(np.sum(weights * y_true)) - violation = max(0.0, weighted_pd_true - float(policy["risk_tolerance"])) - sqrt_alpha = float(np.sqrt(alpha)) + certificate = compute_funded_certificate_metrics( + weights, + outcomes=y_true, + pd_point=pd_point, + pd_high=pd_high, + pd_effective=effective_pd, + alpha=alpha, + risk_tolerance=float(policy["risk_tolerance"]), + pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), + ) realized_total_return = float( np.sum( np.where( @@ -992,11 +835,16 @@ def _exact_policy_alpha( expected_loss_point = float(np.sum(alloc * loan_amounts * pd_point * lgd)) expected_return_net_point = expected_return_gross - expected_loss_point pd_cap_slack = float(solution.get("pd_cap_slack", 0.0)) + risk_excess = round(certificate.realized_risk_tolerance_excess, 6) + empirical_risk_screen = bool(certificate.realized_risk_tolerance_excess <= alpha + 1e-8) + markov_screen = bool(certificate.sqrt_alpha + 1e-8 >= certificate.weighted_miscoverage) return { "alpha": float(alpha), "confidence": float(1.0 - alpha), - "gamma_cp": round(float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))), 6), + "gamma_cp": round(certificate.gamma_cp, 6), + "gamma_internalized": round(certificate.gamma_internalized, 6), + "gamma_residual": round(certificate.gamma_residual, 6), "n_funded": int(solution.get("n_funded", int(np.sum(alloc > 0.01)))), "total_allocated": round(total_allocated, 2), "objective_value": round(float(solution.get("objective_value", 0.0)), 6), @@ -1004,28 +852,32 @@ def _exact_policy_alpha( "expected_loss_point": round(expected_loss_point, 6), "expected_return_net_point": round(expected_return_net_point, 6), "realized_total_return": round(realized_total_return, 6), - "weighted_pd_true": round(weighted_pd_true, 6), - "weighted_pd_constraint_used": round(float(np.sum(weights * effective_pd)), 6), - "weighted_pd_high": round(float(np.sum(weights * pd_high)), 6), - "weighted_pd_point": round(float(np.sum(weights * pd_point)), 6), - "worst_case_pd": round(float(np.sum(weights * pd_high)), 6), - "point_pd": round(float(np.sum(weights * pd_point)), 6), + "weighted_pd_true": round(certificate.weighted_outcome, 6), + "weighted_pd_constraint_used": round(certificate.weighted_pd_effective, 6), + "weighted_pd_high": round(certificate.endpoint_budget, 6), + "weighted_pd_point": round(certificate.weighted_pd_point, 6), + "worst_case_pd": round(certificate.endpoint_budget, 6), + "point_pd": round(certificate.weighted_pd_point, 6), + "endpoint_budget": round(certificate.endpoint_budget, 9), + "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), + "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), + "markov_loss_cap": round(certificate.markov_loss_cap, 9), "tau": float(policy["risk_tolerance"]), - "violation": round(violation, 6), - "weighted_miscoverage_V": round(weighted_miscoverage_v, 6), - "sqrt_alpha": round(sqrt_alpha, 6), - "empirical_coverage_funded": round( - float(1.0 - miscoverage[funded_mask].mean()) if funded_mask.any() else float("nan"), - 4, - ), - "bound_a_expected_violation_leq_alpha": bool(violation <= alpha + 1e-8), + "realized_risk_tolerance_excess": risk_excess, + "violation": risk_excess, + "weighted_miscoverage_V": round(certificate.weighted_miscoverage, 6), + "weighted_coverage_funded": round(certificate.weighted_coverage, 6), + "sqrt_alpha": round(certificate.sqrt_alpha, 6), + "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 4), + "empirical_risk_excess_leq_alpha": empirical_risk_screen, + "bound_a_expected_violation_leq_alpha": empirical_risk_screen, "bound_b_prob_violation_gt_t": round(float(min(1.0, alpha / max(t_eval, 1e-8))), 4), "bound_b_t_eval": float(t_eval), "bound_b_is_vacuous": bool(min(1.0, alpha / max(t_eval, 1e-8)) >= 1.0), - "bound_c_V_leq_sqrt_alpha": bool(sqrt_alpha + 1e-8 >= weighted_miscoverage_v), - "all_bounds_hold": bool( - (violation <= alpha + 1e-8) and (sqrt_alpha + 1e-8 >= weighted_miscoverage_v) - ), + "markov_miscoverage_screen_pass": markov_screen, + "bound_c_V_leq_sqrt_alpha": markov_screen, + "certificate_screen_pass": empirical_risk_screen and markov_screen, + "all_bounds_hold": empirical_risk_screen and markov_screen, "allocator_mode": "exact", "solver_status": str(solution.get("solver_status", "unknown")), "allocator_solver_backend": str(solution.get("solver_backend", policy["solver_backend"])), @@ -1037,12 +889,17 @@ def _exact_policy_alpha( def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) -> pd.DataFrame: if bound_eval.empty: return candidates.copy() + bound_eval = add_policy_aware_bound_columns(bound_eval) grouped = bound_eval.groupby("local_candidate_id", dropna=False) agg = grouped.agg( alpha_exact_pass_count=("all_bounds_hold", "sum"), alpha_exact_check_count=("all_bounds_hold", "size"), alpha_exact_pass_rate=("all_bounds_hold", "mean"), - alpha_max_violation=("violation", "max"), + alpha_max_realized_risk_tolerance_excess=( + "realized_risk_tolerance_excess", + "max", + ), + alpha_max_violation=("realized_risk_tolerance_excess", "max"), alpha_mean_gamma_cp=("gamma_cp", "mean"), alpha_mean_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), alpha_mean_weighted_pd_true=("weighted_pd_true", "mean"), @@ -1065,9 +922,22 @@ def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) - alpha01_exact_pass=("all_bounds_hold", "all"), alpha01_realized_total_return=("realized_total_return", "mean"), alpha01_gamma_cp=("gamma_cp", "mean"), + alpha01_gamma_internalized=("gamma_internalized", "mean"), + alpha01_gamma_residual=("gamma_residual", "mean"), alpha01_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), - alpha01_violation=("violation", "max"), + alpha01_realized_risk_tolerance_excess=( + "realized_risk_tolerance_excess", + "max", + ), + alpha01_violation=("realized_risk_tolerance_excess", "max"), alpha01_weighted_pd_true=("weighted_pd_true", "mean"), + alpha01_weighted_pd_constraint_used=("weighted_pd_constraint_used", "mean"), + alpha01_weighted_pd_high=("weighted_pd_high", "mean"), + alpha01_weighted_pd_point=("weighted_pd_point", "mean"), + alpha01_endpoint_budget=("endpoint_budget", "mean"), + alpha01_endpoint_budget_upper=("endpoint_budget_upper", "mean"), + alpha01_markov_loss_threshold=("markov_loss_threshold", "mean"), + alpha01_markov_loss_cap=("markov_loss_cap", "mean"), alpha01_empirical_coverage_funded=("empirical_coverage_funded", "mean"), alpha01_n_funded=("n_funded", "mean"), ) @@ -1084,11 +954,6 @@ def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) - work["return_floor_surplus"] = ( work["alpha01_realized_total_return"].fillna(float("-inf")) - DECLARED_RETURN_FLOOR ) - alpha01_gamma = pd.to_numeric(work["alpha01_gamma_cp"], errors="coerce") - risk = pd.to_numeric(work["risk_tolerance"], errors="coerce") - gamma = pd.to_numeric(work["gamma"], errors="coerce") - work["alpha01_endpoint_budget_upper"] = risk + (1.0 - gamma) * alpha01_gamma - work["alpha01_markov_loss_cap"] = work["alpha01_endpoint_budget_upper"] + float(np.sqrt(0.01)) work = work.sort_values( by=[ "alpha01_exact_pass", @@ -1101,178 +966,171 @@ def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) - ascending=[False, False, False, False, True, True], kind="mergesort", ).reset_index(drop=True) - work.insert(0, "claim_rank", range(1, len(work) + 1)) + work.insert(0, "claim_rank", np.arange(1, len(work) + 1, dtype=int)) return work -def _claim_summary( - leaderboard: pd.DataFrame, - bound_eval: pd.DataFrame, - *, - alpha_grid: list[float] | None = None, -) -> dict[str, Any]: - leaderboard = leaderboard.copy() - if "alpha01_endpoint_budget_upper" not in leaderboard.columns: - alpha01_gamma = pd.to_numeric(leaderboard["alpha01_gamma_cp"], errors="coerce") - risk = pd.to_numeric(leaderboard["risk_tolerance"], errors="coerce") - gamma = pd.to_numeric(leaderboard["gamma"], errors="coerce") - leaderboard["alpha01_endpoint_budget_upper"] = risk + (1.0 - gamma) * alpha01_gamma - if "alpha01_markov_loss_cap" not in leaderboard.columns: - leaderboard["alpha01_markov_loss_cap"] = pd.to_numeric( - leaderboard["alpha01_endpoint_budget_upper"], errors="coerce" +def _ensure_claim_summary_columns(leaderboard: pd.DataFrame) -> pd.DataFrame: + work = leaderboard.copy() + if "alpha01_endpoint_budget" not in work.columns and "alpha01_weighted_pd_high" in work.columns: + work["alpha01_endpoint_budget"] = pd.to_numeric( + work["alpha01_weighted_pd_high"], errors="coerce" + ) + if "alpha01_endpoint_budget_upper" not in work.columns: + if { + "alpha01_weighted_pd_high", + "alpha01_weighted_pd_constraint_used", + }.issubset(work.columns): + residual = pd.to_numeric( + work["alpha01_weighted_pd_high"], errors="coerce" + ) - pd.to_numeric(work["alpha01_weighted_pd_constraint_used"], errors="coerce") + work["alpha01_gamma_residual"] = residual.clip(lower=0.0) + work["alpha01_endpoint_budget_upper"] = pd.to_numeric( + work["risk_tolerance"], errors="coerce" + ) + residual.clip(lower=0.0) + else: + alpha01_gamma = pd.to_numeric(work["alpha01_gamma_cp"], errors="coerce") + risk = pd.to_numeric(work["risk_tolerance"], errors="coerce") + gamma = pd.to_numeric(work["gamma"], errors="coerce") + work["alpha01_endpoint_budget_upper"] = risk + (1.0 - gamma) * alpha01_gamma + if "alpha01_markov_loss_threshold" not in work.columns and "alpha01_endpoint_budget" in work: + work["alpha01_markov_loss_threshold"] = pd.to_numeric( + work["alpha01_endpoint_budget"], errors="coerce" ) + float(np.sqrt(0.01)) - eligible = leaderboard[ - leaderboard["alpha01_exact_pass"].fillna(False).astype(bool) - & leaderboard["all_alpha_pass"].fillna(False).astype(bool) - ].copy() - if "return_floor_surplus" not in leaderboard.columns: - if "champion_return_surplus" in leaderboard.columns: - leaderboard["return_floor_surplus"] = leaderboard["champion_return_surplus"] + if "alpha01_markov_loss_cap" not in work.columns: + work["alpha01_markov_loss_cap"] = pd.to_numeric( + work["alpha01_endpoint_budget_upper"], errors="coerce" + ) + float(np.sqrt(0.01)) + if "return_floor_surplus" not in work.columns: + if "champion_return_surplus" in work.columns: + work["return_floor_surplus"] = work["champion_return_surplus"] else: - leaderboard["return_floor_surplus"] = ( - leaderboard["alpha01_realized_total_return"].fillna(float("-inf")) - - DECLARED_RETURN_FLOOR + work["return_floor_surplus"] = ( + work["alpha01_realized_total_return"].fillna(float("-inf")) - DECLARED_RETURN_FLOOR ) - above_return_floor = eligible[ - eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR + return work + + +def _all_alpha_eligible(frame: pd.DataFrame) -> pd.DataFrame: + return frame[ + frame["alpha01_exact_pass"].fillna(False).astype(bool) + & frame["all_alpha_pass"].fillna(False).astype(bool) ].copy() - def row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: - if frame.empty: - return None - row = frame.iloc[0] - fields = [ - "claim_rank", - "local_candidate_id", - "local_family", - "anchor_rank", - "source_reason", - "risk_tolerance", - "policy_mode", - "gamma", - "delta_cap_quantile", - "tail_focus_quantile", - "uncertainty_aversion", - "alpha01_realized_total_return", - "return_floor_surplus", - "alpha01_gamma_cp", - "alpha01_weighted_miscoverage_V", - "alpha01_endpoint_budget_upper", - "alpha01_markov_loss_cap", - "alpha01_weighted_pd_true", - "alpha01_empirical_coverage_funded", - "alpha_exact_pass_count", - "alpha_exact_check_count", - "alpha_mean_gamma_cp", - "alpha_mean_weighted_miscoverage_V", - "return_score", - "bound_score", - "v_score", - "ijds_balanced_score", - "n_funded_mean", - "allocator_backends", - ] - return { - field: row[field].item() if hasattr(row[field], "item") else row[field] - for field in fields - if field in row.index - } - max_return = row_payload(eligible.sort_values("alpha01_realized_total_return", ascending=False)) - best_gamma = row_payload( - above_return_floor.sort_values( - ["alpha01_gamma_cp", "alpha01_realized_total_return"], - ascending=[True, False], - ) - ) - best_v = row_payload( - above_return_floor.sort_values( - ["alpha01_weighted_miscoverage_V", "alpha01_realized_total_return"], - ascending=[True, False], - ) - ) +def _row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: + if frame.empty: + return None + row = frame.iloc[0] + return { + field: row[field].item() if hasattr(row[field], "item") else row[field] + for field in CLAIM_ROW_FIELDS + if field in row.index + } + + +def _add_normalized_score( + frame: pd.DataFrame, + *, + source: str, + target: str, + higher_better: bool, +) -> None: + vals = pd.to_numeric(frame[source], errors="coerce") + lo, hi = float(vals.min()), float(vals.max()) + if hi <= lo: + frame[target] = 1.0 + elif higher_better: + frame[target] = (vals - lo) / (hi - lo) + else: + frame[target] = (hi - vals) / (hi - lo) + + +def _balanced_claim_candidates(above_return_floor: pd.DataFrame) -> pd.DataFrame: balanced = above_return_floor.copy() - if not balanced.empty: - for source, target, higher_better in [ - ("alpha01_realized_total_return", "return_score", True), - ("alpha01_markov_loss_cap", "bound_score", False), - ("alpha01_weighted_miscoverage_V", "v_score", False), - ]: - vals = pd.to_numeric(balanced[source], errors="coerce") - lo, hi = float(vals.min()), float(vals.max()) - if hi <= lo: - balanced[target] = 1.0 - elif higher_better: - balanced[target] = (vals - lo) / (hi - lo) - else: - balanced[target] = (hi - vals) / (hi - lo) - balanced["ijds_balanced_score"] = ( - 0.40 * balanced["return_score"] - + 0.40 * balanced["bound_score"] - + 0.20 * balanced["v_score"] + if balanced.empty: + return balanced + for source, target, higher_better in [ + ("alpha01_realized_total_return", "return_score", True), + ("alpha01_markov_loss_cap", "bound_score", False), + ("alpha01_weighted_miscoverage_V", "v_score", False), + ]: + _add_normalized_score( + balanced, + source=source, + target=target, + higher_better=higher_better, ) - balanced_claim = row_payload( - balanced.sort_values("ijds_balanced_score", ascending=False) - if not balanced.empty - else balanced + balanced["ijds_balanced_score"] = ( + 0.40 * balanced["return_score"] + + 0.40 * balanced["bound_score"] + + 0.20 * balanced["v_score"] ) + return balanced + +def _family_claim_summary(leaderboard: pd.DataFrame) -> dict[str, Any]: by_family: dict[str, Any] = {} - if not leaderboard.empty: - for family, frame in leaderboard.groupby("local_family", dropna=False): - fam_eligible = frame[ - frame["alpha01_exact_pass"].fillna(False).astype(bool) - & frame["all_alpha_pass"].fillna(False).astype(bool) - ] - by_family[str(family)] = { - "n_policies": int(len(frame)), - "n_all_alpha_passers": int(len(fam_eligible)), - "all_alpha_pass_rate": float(len(fam_eligible) / max(len(frame), 1)), - "best_return": float(fam_eligible["alpha01_realized_total_return"].max()) - if not fam_eligible.empty - else None, - "min_gamma_cp_above_return_floor": float( - fam_eligible.loc[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR, - "alpha01_gamma_cp", - ].min() - ) - if not fam_eligible[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR - ].empty - else None, - "min_v_above_return_floor": float( - fam_eligible.loc[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR, - "alpha01_weighted_miscoverage_V", - ].min() - ) - if not fam_eligible[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR - ].empty - else None, - } + if leaderboard.empty: + return by_family + for family, frame in leaderboard.groupby("local_family", dropna=False): + fam_eligible = _all_alpha_eligible(frame) + fam_above_floor = fam_eligible[ + fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR + ] + by_family[str(family)] = { + "n_policies": int(len(frame)), + "n_all_alpha_passers": int(len(fam_eligible)), + "all_alpha_pass_rate": float(len(fam_eligible) / max(len(frame), 1)), + "best_return": float(fam_eligible["alpha01_realized_total_return"].max()) + if not fam_eligible.empty + else None, + "min_gamma_cp_above_return_floor": float(fam_above_floor["alpha01_gamma_cp"].min()) + if not fam_above_floor.empty + else None, + "min_v_above_return_floor": float( + fam_above_floor["alpha01_weighted_miscoverage_V"].min() + ) + if not fam_above_floor.empty + else None, + } + return by_family + +def _alpha_claim_summary(bound_eval: pd.DataFrame) -> dict[str, Any]: by_alpha: dict[str, Any] = {} - if not bound_eval.empty: - for alpha, frame in bound_eval.groupby("alpha", dropna=False): - by_alpha[str(float(alpha))] = { - "n_checks": int(len(frame)), - "pass_rate": float(frame["all_bounds_hold"].fillna(False).mean()), - "max_violation": float(frame["violation"].max()), - "mean_gamma_cp": float(frame["gamma_cp"].mean()), - "mean_weighted_miscoverage_V": float(frame["weighted_miscoverage_V"].mean()), - } - - alpha_values = ( - [float(value) for value in alpha_grid] - if alpha_grid is not None - else sorted(float(value) for value in bound_eval["alpha"].dropna().unique()) - if "alpha" in bound_eval - else list(DEFAULT_ALPHA_GRID) + if bound_eval.empty: + return by_alpha + risk_excess_column = ( + "realized_risk_tolerance_excess" + if "realized_risk_tolerance_excess" in bound_eval.columns + else "violation" ) - alpha_values = sorted(dict.fromkeys(alpha_values)) - finite_grid_policy = { + for alpha, frame in bound_eval.groupby("alpha", dropna=False): + alpha_value = float(str(alpha)) + by_alpha[str(alpha_value)] = { + "n_checks": int(len(frame)), + "pass_rate": float(frame["all_bounds_hold"].fillna(False).mean()), + "max_realized_risk_tolerance_excess": float(frame[risk_excess_column].max()), + "max_violation": float(frame[risk_excess_column].max()), + "mean_gamma_cp": float(frame["gamma_cp"].mean()), + "mean_weighted_miscoverage_V": float(frame["weighted_miscoverage_V"].mean()), + } + return by_alpha + + +def _claim_alpha_values(bound_eval: pd.DataFrame, alpha_grid: list[float] | None) -> list[float]: + if alpha_grid is not None: + values = [float(value) for value in alpha_grid] + elif "alpha" in bound_eval: + values = sorted(float(value) for value in bound_eval["alpha"].dropna().unique()) + else: + values = list(DEFAULT_ALPHA_GRID) + return sorted(dict.fromkeys(values)) + + +def _finite_grid_policy(alpha_values: list[float]) -> dict[str, Any]: + return { "alpha_grid": alpha_values, "alpha_grid_size": int(len(alpha_values)), "alpha_grid_semantics": ( @@ -1284,7 +1142,10 @@ def row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: "not a continuous robust region" ), } - claim_selection_protocol = { + + +def _claim_selection_protocol() -> dict[str, Any]: + return { "body_default": "balanced_return_bound_claim", "frontier_endpoints": [ "max_return_claim", @@ -1308,12 +1169,48 @@ def row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: ), } + +def _claim_summary( + leaderboard: pd.DataFrame, + bound_eval: pd.DataFrame, + *, + alpha_grid: list[float] | None = None, +) -> dict[str, Any]: + leaderboard = _ensure_claim_summary_columns(leaderboard) + eligible = _all_alpha_eligible(leaderboard) + above_return_floor = eligible[ + eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR + ].copy() + + max_return = _row_payload( + eligible.sort_values("alpha01_realized_total_return", ascending=False) + ) + best_gamma = _row_payload( + above_return_floor.sort_values( + ["alpha01_gamma_cp", "alpha01_realized_total_return"], + ascending=[True, False], + ) + ) + best_v = _row_payload( + above_return_floor.sort_values( + ["alpha01_weighted_miscoverage_V", "alpha01_realized_total_return"], + ascending=[True, False], + ) + ) + balanced = _balanced_claim_candidates(above_return_floor) + balanced_claim = _row_payload( + balanced.sort_values("ijds_balanced_score", ascending=False) + if not balanced.empty + else balanced + ) + alpha_values = _claim_alpha_values(bound_eval, alpha_grid) + return { "schema_version": SCHEMA_VERSION, "generated_at_utc": datetime.now(tz=UTC).isoformat(), "declared_return_floor": DECLARED_RETURN_FLOOR, - "finite_grid_policy": finite_grid_policy, - "claim_selection_protocol": claim_selection_protocol, + "finite_grid_policy": _finite_grid_policy(alpha_values), + "claim_selection_protocol": _claim_selection_protocol(), "n_policies": int(len(leaderboard)), "n_all_alpha_passers": int(len(eligible)), "n_all_alpha_passers_above_return_floor": int(len(above_return_floor)), @@ -1321,8 +1218,8 @@ def row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: "best_gamma_cp_return_floor_claim": best_gamma, "best_weighted_miscoverage_return_floor_claim": best_v, "balanced_return_bound_claim": balanced_claim, - "by_family": by_family, - "by_alpha": by_alpha, + "by_family": _family_claim_summary(leaderboard), + "by_alpha": _alpha_claim_summary(bound_eval), "interpretation": { "max_return_claim": "Use when the paper emphasizes certified economic return.", "best_gamma_cp_return_floor_claim": "Use when the paper emphasizes a tighter conformal robustness budget while preserving the declared return floor.", @@ -1373,7 +1270,7 @@ def _write_status( ) -def main(argv: list[str] | None = None) -> int: +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument("--run-tag", default="pool93_ijds_local_refine_stage1") parser.add_argument( @@ -1419,9 +1316,10 @@ def main(argv: list[str] | None = None) -> int: default=0, help="Debug/smoke option: keep only the first N generated policies when positive.", ) - args = parser.parse_args(argv) + return parser - run_tag = str(args.run_tag).strip().replace("/", "_") + +def _resolve_paths(args: argparse.Namespace, *, run_tag: str) -> Pool93Paths: output_dir = ( Path(args.output_dir) if str(args.output_dir).strip() @@ -1432,44 +1330,81 @@ def main(argv: list[str] | None = None) -> int: if str(args.model_dir).strip() else ROOT / "models/experiments/champion_reopen" / run_tag / "portfolio" ) - output_dir.mkdir(parents=True, exist_ok=True) - model_dir.mkdir(parents=True, exist_ok=True) - checkpoint_dir = model_dir / "runtime_checkpoints" - checkpoint_dir.mkdir(parents=True, exist_ok=True) - - status_path = model_dir / "runtime_status.json" - candidates_path = output_dir / "pool93_ijds_local_refinement_candidates.parquet" - bound_eval_path = output_dir / "pool93_ijds_local_refinement_bound_eval.parquet" - leaderboard_path = output_dir / "pool93_ijds_local_refinement_leaderboard.parquet" - claim_summary_path = model_dir / "pool93_ijds_local_refinement_claim_summary.json" - manifest_path = model_dir / "pool93_ijds_local_refinement_manifest.json" + return Pool93Paths( + output_dir=output_dir, + model_dir=model_dir, + checkpoint_dir=model_dir / "runtime_checkpoints", + status_path=model_dir / "runtime_status.json", + candidates_path=output_dir / "pool93_ijds_local_refinement_candidates.parquet", + bound_eval_path=output_dir / "pool93_ijds_local_refinement_bound_eval.parquet", + leaderboard_path=output_dir / "pool93_ijds_local_refinement_leaderboard.parquet", + claim_summary_path=model_dir / "pool93_ijds_local_refinement_claim_summary.json", + manifest_path=model_dir / "pool93_ijds_local_refinement_manifest.json", + ) - source_bound_eval = Path(args.source_bound_eval) - source_selection = Path(args.source_selection) + +def _ensure_pool93_dirs(paths: Pool93Paths) -> None: + paths.output_dir.mkdir(parents=True, exist_ok=True) + paths.model_dir.mkdir(parents=True, exist_ok=True) + paths.checkpoint_dir.mkdir(parents=True, exist_ok=True) + + +def _conformal_intervals_from_selection( + *, + explicit_path: str, + source_selection: Path, +) -> str: + if str(explicit_path).strip(): + return str(explicit_path).strip() source_selection_payload = json.loads(source_selection.read_text(encoding="utf-8")) - conformal_intervals_path = str(args.conformal_intervals_path).strip() or str( - ROOT / source_selection_payload["conformal_intervals_path"] - ) - alpha_grid = _coerce_float_grid(args.alpha_grid, DEFAULT_ALPHA_GRID) - anchor_ranks = _coerce_int_grid(args.anchor_ranks, [96, 219, 223]) + return str(ROOT / source_selection_payload["conformal_intervals_path"]) - if candidates_path.exists(): - candidates = pd.read_parquet(candidates_path) - logger.info("Reusing candidate manifest: {} rows from {}", len(candidates), candidates_path) - else: - anchors = _source_anchor_rows(source_bound_eval, anchor_ranks) - candidates = _generate_candidate_grid( - anchors, - profile=str(args.profile), - solver_backend=str(args.solver_backend), + +def _load_or_generate_candidates( + *, + args: argparse.Namespace, + paths: Pool93Paths, + source_bound_eval: Path, + anchor_ranks: list[int], +) -> pd.DataFrame: + if paths.candidates_path.exists(): + candidates = pd.read_parquet(paths.candidates_path) + logger.info( + "Reusing candidate manifest: {} rows from {}", + len(candidates), + paths.candidates_path, ) - if int(args.candidate_limit) > 0: - candidates = candidates.head(int(args.candidate_limit)).copy().reset_index(drop=True) - candidates["local_candidate_id"] = range(1, len(candidates) + 1) - atomic_write_parquet(candidates, candidates_path, index=False) - logger.info("Wrote candidate manifest: {} policies to {}", len(candidates), candidates_path) + return candidates + anchors = _source_anchor_rows(source_bound_eval, anchor_ranks) + candidates = _generate_candidate_grid( + anchors, + profile=str(args.profile), + solver_backend=str(args.solver_backend), + ) + if int(args.candidate_limit) > 0: + candidates = candidates.head(int(args.candidate_limit)).copy().reset_index(drop=True) + candidates["local_candidate_id"] = np.arange(1, len(candidates) + 1, dtype=int) + atomic_write_parquet(candidates, paths.candidates_path, index=False) + logger.info( + "Wrote candidate manifest: {} policies to {}", + len(candidates), + paths.candidates_path, + ) + return candidates + - manifest = { +def _manifest_payload( + *, + args: argparse.Namespace, + paths: Pool93Paths, + run_tag: str, + source_bound_eval: Path, + source_selection: Path, + conformal_intervals_path: str, + anchor_ranks: list[int], + alpha_grid: list[float], +) -> dict[str, Any]: + return { "schema_version": SCHEMA_VERSION, "generated_at_utc": datetime.now(tz=UTC).isoformat(), "run_tag": run_tag, @@ -1487,28 +1422,192 @@ def main(argv: list[str] | None = None) -> int: "random_state": int(args.random_state), "checkpoint_every": int(args.checkpoint_every), "parallel_workers": int(args.parallel_workers), - "candidates_path": str(candidates_path), - "bound_eval_path": str(bound_eval_path), - "leaderboard_path": str(leaderboard_path), - "claim_summary_path": str(claim_summary_path), + "candidates_path": str(paths.candidates_path), + "bound_eval_path": str(paths.bound_eval_path), + "leaderboard_path": str(paths.leaderboard_path), + "claim_summary_path": str(paths.claim_summary_path), } - atomic_write_json(manifest_path, manifest) + +def _load_partial_bound_eval(path: Path) -> tuple[pd.DataFrame, set[tuple[int, float]]]: partial = pd.DataFrame() - if bound_eval_path.exists(): - partial = pd.read_parquet(bound_eval_path) + if path.exists(): + partial = pd.read_parquet(path) if not partial.empty: partial = partial.drop_duplicates( ["local_candidate_id", "alpha"], keep="last", ).reset_index(drop=True) logger.info("Resuming local refinement from {} rows", len(partial)) - completed_keys = set() - if not partial.empty: - completed_keys = { - (int(row.local_candidate_id), float(row.alpha)) - for row in partial.itertuples(index=False) - } + completed_keys = { + (int(row["local_candidate_id"]), float(row["alpha"])) + for row in partial.to_dict(orient="records") + } + return partial, completed_keys + + +def _pending_refinement_tasks( + *, + candidates: pd.DataFrame, + alpha_grid: list[float], + completed_keys: set[tuple[int, float]], +) -> list[tuple[dict[str, Any], float]]: + return [ + (candidate, float(alpha)) + for candidate in candidates.to_dict(orient="records") + for alpha in alpha_grid + if (int(candidate["local_candidate_id"]), float(alpha)) not in completed_keys + ] + + +def _persist_refinement_progress( + *, + paths: Pool93Paths, + candidates: pd.DataFrame, + rows: list[dict[str, Any]], + alpha_grid: list[float], +) -> None: + bound_eval = pd.DataFrame(rows) + atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) + leaderboard = _aggregate_leaderboard(candidates, bound_eval) + atomic_write_parquet(leaderboard, paths.leaderboard_path, index=False) + atomic_write_json( + paths.claim_summary_path, + _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid), + ) + + +def _run_serial_refinement( + *, + pending_tasks: list[tuple[dict[str, Any], float]], + aligned: pd.DataFrame, + budget: float, + t_eval: float, + exact_threads: int, + record_result: Any, +) -> None: + for candidate, alpha in pending_tasks: + policy = {field: candidate[field] for field in SEMANTIC_POLICY_FIELDS} + result = _exact_policy_alpha( + aligned, + policy=policy, + alpha=float(alpha), + budget=float(budget), + t_eval=float(t_eval), + threads=int(exact_threads), + ) + record_result(candidate, alpha, result) + + +def _run_parallel_refinement( + *, + pending_tasks: list[tuple[dict[str, Any], float]], + aligned: pd.DataFrame, + parallel_workers: int, + budget: float, + t_eval: float, + exact_threads: int, + persist_progress: Any, + record_result: Any, +) -> None: + logger.info( + "Running exact refinement with {} parallel workers and {} solver thread(s) per worker", + parallel_workers, + int(exact_threads), + ) + mp_context = mp.get_context("fork") if sys.platform != "win32" else None + max_in_flight = max(parallel_workers, parallel_workers * 2) + next_task_idx = 0 + futures: dict[Any, tuple[dict[str, Any], float]] = {} + with ProcessPoolExecutor( + max_workers=parallel_workers, + mp_context=mp_context, + initializer=_init_exact_worker, + initargs=(aligned,), + ) as executor: + while next_task_idx < len(pending_tasks) or futures: + while next_task_idx < len(pending_tasks) and len(futures) < max_in_flight: + candidate, alpha = pending_tasks[next_task_idx] + future = executor.submit( + _exact_policy_alpha_task, + candidate, + alpha, + float(budget), + float(t_eval), + int(exact_threads), + ) + futures[future] = (candidate, alpha) + next_task_idx += 1 + done, _ = wait(futures, return_when=FIRST_COMPLETED) + for future in done: + candidate, alpha = futures.pop(future) + try: + result = future.result() + except Exception: + persist_progress() + raise + result_only = { + key: value + for key, value in result.items() + if key not in candidate or key in {"alpha", "confidence"} + } + record_result(candidate, alpha, result_only) + + +def _write_final_outputs( + *, + paths: Pool93Paths, + candidates: pd.DataFrame, + rows: list[dict[str, Any]], + alpha_grid: list[float], +) -> tuple[pd.DataFrame, dict[str, Any]]: + bound_eval = pd.DataFrame(rows) + atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) + leaderboard = _aggregate_leaderboard(candidates, bound_eval) + atomic_write_parquet(leaderboard, paths.leaderboard_path, index=False) + claim_summary = _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid) + atomic_write_json(paths.claim_summary_path, claim_summary) + return leaderboard, claim_summary + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + run_tag = str(args.run_tag).strip().replace("/", "_") + paths = _resolve_paths(args, run_tag=run_tag) + _ensure_pool93_dirs(paths) + + source_bound_eval = Path(args.source_bound_eval) + source_selection = Path(args.source_selection) + conformal_intervals_path = _conformal_intervals_from_selection( + explicit_path=args.conformal_intervals_path, + source_selection=source_selection, + ) + alpha_grid = _coerce_float_grid(args.alpha_grid, DEFAULT_ALPHA_GRID) + anchor_ranks = _coerce_int_grid(args.anchor_ranks, [96, 219, 223]) + + candidates = _load_or_generate_candidates( + args=args, + paths=paths, + source_bound_eval=source_bound_eval, + anchor_ranks=anchor_ranks, + ) + atomic_write_json( + paths.manifest_path, + _manifest_payload( + args=args, + paths=paths, + run_tag=run_tag, + source_bound_eval=source_bound_eval, + source_selection=source_selection, + conformal_intervals_path=conformal_intervals_path, + anchor_ranks=anchor_ranks, + alpha_grid=alpha_grid, + ), + ) + + partial, completed_keys = _load_partial_bound_eval(paths.bound_eval_path) rows: list[dict[str, Any]] = partial.to_dict(orient="records") if not partial.empty else [] total_checks = int(len(candidates) * len(alpha_grid)) @@ -1516,7 +1615,7 @@ def main(argv: list[str] | None = None) -> int: initial_completed = int(len(completed_keys)) _write_status( run_tag=run_tag, - status_path=status_path, + status_path=paths.status_path, start_monotonic=start, completed=len(completed_keys), total=total_checks, @@ -1532,15 +1631,14 @@ def main(argv: list[str] | None = None) -> int: random_state=int(args.random_state), ) logger.info("Loaded aligned full universe: {} rows", len(aligned)) + completed = len(completed_keys) def persist_progress() -> None: - bound_eval = pd.DataFrame(rows) - atomic_write_parquet(bound_eval, bound_eval_path, index=False) - leaderboard = _aggregate_leaderboard(candidates, bound_eval) - atomic_write_parquet(leaderboard, leaderboard_path, index=False) - atomic_write_json( - claim_summary_path, - _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid), + _persist_refinement_progress( + paths=paths, + candidates=candidates, + rows=rows, + alpha_grid=alpha_grid, ) def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any]) -> None: @@ -1554,7 +1652,7 @@ def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any completed_keys.add((int(candidate["local_candidate_id"]), float(alpha))) _write_status( run_tag=run_tag, - status_path=status_path, + status_path=paths.status_path, start_monotonic=start, completed=completed, total=total_checks, @@ -1574,81 +1672,44 @@ def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any if completed % max(1, int(args.checkpoint_every)) == 0: persist_progress() - completed = len(completed_keys) - pending_tasks: list[tuple[dict[str, Any], float]] = [] - for candidate in candidates.to_dict(orient="records"): - for alpha in alpha_grid: - key = (int(candidate["local_candidate_id"]), float(alpha)) - if key not in completed_keys: - pending_tasks.append((candidate, float(alpha))) + pending_tasks = _pending_refinement_tasks( + candidates=candidates, + alpha_grid=alpha_grid, + completed_keys=completed_keys, + ) parallel_workers = max(1, int(args.parallel_workers)) if parallel_workers <= 1: - for candidate, alpha in pending_tasks: - policy = {field: candidate[field] for field in SEMANTIC_POLICY_FIELDS} - result = _exact_policy_alpha( - aligned, - policy=policy, - alpha=float(alpha), - budget=float(args.budget), - t_eval=float(args.t_eval), - threads=int(args.exact_threads), - ) - record_result(candidate, alpha, result) + _run_serial_refinement( + pending_tasks=pending_tasks, + aligned=aligned, + budget=float(args.budget), + t_eval=float(args.t_eval), + exact_threads=int(args.exact_threads), + record_result=record_result, + ) else: - logger.info( - "Running exact refinement with {} parallel workers and {} solver thread(s) per worker", - parallel_workers, - int(args.exact_threads), + _run_parallel_refinement( + pending_tasks=pending_tasks, + aligned=aligned, + parallel_workers=parallel_workers, + budget=float(args.budget), + t_eval=float(args.t_eval), + exact_threads=int(args.exact_threads), + persist_progress=persist_progress, + record_result=record_result, ) - mp_context = mp.get_context("fork") if sys.platform != "win32" else None - max_in_flight = max(parallel_workers, parallel_workers * 2) - next_task_idx = 0 - futures: dict[Any, tuple[dict[str, Any], float]] = {} - with ProcessPoolExecutor( - max_workers=parallel_workers, - mp_context=mp_context, - initializer=_init_exact_worker, - initargs=(aligned,), - ) as executor: - while next_task_idx < len(pending_tasks) or futures: - while next_task_idx < len(pending_tasks) and len(futures) < max_in_flight: - candidate, alpha = pending_tasks[next_task_idx] - future = executor.submit( - _exact_policy_alpha_task, - candidate, - alpha, - float(args.budget), - float(args.t_eval), - int(args.exact_threads), - ) - futures[future] = (candidate, alpha) - next_task_idx += 1 - done, _ = wait(futures, return_when=FIRST_COMPLETED) - for future in done: - candidate, alpha = futures.pop(future) - try: - result = future.result() - except Exception: - persist_progress() - raise - result_only = { - key: value - for key, value in result.items() - if key not in candidate or key in {"alpha", "confidence"} - } - record_result(candidate, alpha, result_only) - bound_eval = pd.DataFrame(rows) - atomic_write_parquet(bound_eval, bound_eval_path, index=False) - leaderboard = _aggregate_leaderboard(candidates, bound_eval) - atomic_write_parquet(leaderboard, leaderboard_path, index=False) - claim_summary = _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid) - atomic_write_json(claim_summary_path, claim_summary) + leaderboard, claim_summary = _write_final_outputs( + paths=paths, + candidates=candidates, + rows=rows, + alpha_grid=alpha_grid, + ) _write_status( run_tag=run_tag, - status_path=status_path, + status_path=paths.status_path, start_monotonic=start, completed=total_checks, total=total_checks, @@ -1661,8 +1722,8 @@ def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any "n_all_alpha_passers_above_return_floor": int( claim_summary["n_all_alpha_passers_above_return_floor"] ), - "claim_summary_path": str(claim_summary_path), - "leaderboard_path": str(leaderboard_path), + "claim_summary_path": str(paths.claim_summary_path), + "leaderboard_path": str(paths.leaderboard_path), }, ) write_runtime_checkpoint( @@ -1671,12 +1732,12 @@ def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any { "run_tag": run_tag, "completed_at_utc": datetime.now(tz=UTC).isoformat(), - "claim_summary_path": str(claim_summary_path), - "leaderboard_path": str(leaderboard_path), + "claim_summary_path": str(paths.claim_summary_path), + "leaderboard_path": str(paths.leaderboard_path), }, - checkpoint_dir=checkpoint_dir, + checkpoint_dir=paths.checkpoint_dir, ) - logger.info("Local IJDS refinement complete: {}", claim_summary_path) + logger.info("Local IJDS refinement complete: {}", paths.claim_summary_path) return 0 diff --git a/scripts/search/run_portfolio_bound_aware_search.py b/scripts/search/run_portfolio_bound_aware_search.py index 3c626d6..cb4be29 100644 --- a/scripts/search/run_portfolio_bound_aware_search.py +++ b/scripts/search/run_portfolio_bound_aware_search.py @@ -10,6 +10,7 @@ import sys import time from collections.abc import Callable, Mapping +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -36,6 +37,9 @@ _load_aligned_dataset, _validate_single_alpha, ) +from src.optimization.certificate_semantics import ( # noqa: E402 + IJDS_DECLARED_ALPHA_GRID_CSV, +) from src.utils.pipeline_runtime import ( # noqa: E402 atomic_write_json, atomic_write_parquet, @@ -60,6 +64,91 @@ ] +@dataclass(frozen=True) +class BoundAwarePaths: + output_dir: Path + model_dir: Path + status_path: Path + checkpoint_dir: Path + resource_path: Path + gpu_csv_path: Path + frontier_raw_path: Path + frontier_path: Path + shortlist_path: Path + shortlist_exact_path: Path + bound_eval_path: Path + region_summary_path: Path + selection_path: Path + exact_context_path: Path + + +@dataclass(frozen=True) +class BoundAwareGridSpec: + risk_values: list[float] + aversion_values: list[float] + gamma_values: list[float] + delta_cap_quantiles: list[float] + tail_focus_quantiles: list[float] + alpha_grid: list[float] + random_states: list[int] + exact_random_states: list[int] + exact_max_candidates: int + budget_profiles: list[dict[str, Any]] + policy_modes: list[str] + incumbent_risk_neighbors: list[float] + incumbent_gamma_neighbors: list[float] + incumbent_policy_modes: list[str] + + @property + def policy_grid_count(self) -> int: + return _policy_grid_size( + gamma_values=self.gamma_values, + delta_cap_quantiles=self.delta_cap_quantiles, + tail_focus_quantiles=self.tail_focus_quantiles, + aversion_values=self.aversion_values, + budget_profiles=self.budget_profiles, + policy_modes=self.policy_modes, + ) + + @property + def frontier_total_units(self) -> int: + return int(len(self.random_states) * len(self.risk_values) * (1 + self.policy_grid_count)) + + def bound_total_checks(self, shortlist_size: int) -> int: + return int(shortlist_size * len(self.alpha_grid) * len(self.exact_random_states)) + + +@dataclass(frozen=True) +class BoundAwareExecutionSpec: + run_label: str + paths: BoundAwarePaths + grid: BoundAwareGridSpec + incumbent_policy: dict[str, Any] + exact_python_executable: str + exact_helper_script: Path + backend_validation: dict[str, Any] | None + cuopt_parameters: dict[str, Any] + + +@dataclass +class BoundAwareRunContext: + args: argparse.Namespace + spec: BoundAwareExecutionSpec + tracker: _ProgressTracker + search_space: dict[str, Any] + resource_payload: dict[str, Any] + + +@dataclass(frozen=True) +class BoundAwareFrontierState: + frontier_raw: pd.DataFrame + frontier: pd.DataFrame + shortlist: pd.DataFrame + bound_total_checks: int + shortlist_extra: dict[str, Any] + selection_context: dict[str, Any] + + def _env_int(name: str, fallback: int) -> int: raw = os.environ.get(name) if raw is None: @@ -83,6 +172,10 @@ def _coerce_int_csv(raw: str | None, *, fallback: int) -> list[int]: return values or [int(fallback)] +def _coerce_str_csv(raw: object) -> list[str]: + return [part.strip() for part in str(raw).split(",") if part.strip()] + + def _float_token(value: Any) -> float: return round(float(value), 10) @@ -119,6 +212,23 @@ def _policy_from_row( } +def _normalized_policy_mode_filter(policy_modes: list[str] | None) -> set[str]: + return {str(mode).strip() for mode in (policy_modes or []) if str(mode).strip()} + + +def _mode_quantile_pairs( + *, + mode: str, + delta_cap_quantiles: list[float], + tail_focus_quantiles: list[float], +) -> list[tuple[float, float]]: + if mode == "blended_uncertainty": + return [(1.0, 1.0)] + if mode == "capped_blended_uncertainty": + return [(float(delta_cap_quantile), 1.0) for delta_cap_quantile in delta_cap_quantiles] + return [(1.0, float(tail_focus_quantile)) for tail_focus_quantile in tail_focus_quantiles] + + def _targeted_policy_grid( *, gamma_values: list[float], @@ -126,52 +236,25 @@ def _targeted_policy_grid( tail_focus_quantiles: list[float], policy_modes: list[str] | None = None, ) -> list[tuple[str, float, float, float]]: - allowed = {str(mode).strip() for mode in (policy_modes or []) if str(mode).strip()} - use_all = not allowed - grid: list[tuple[str, float, float, float]] = [] - for gamma in gamma_values: - if use_all or "blended_uncertainty" in allowed: - grid.append(("blended_uncertainty", float(gamma), 1.0, 1.0)) - if use_all or "capped_blended_uncertainty" in allowed: - for delta_cap_quantile in delta_cap_quantiles: - grid.append( - ( - "capped_blended_uncertainty", - float(gamma), - float(delta_cap_quantile), - 1.0, - ) - ) - if use_all or "tail_blended_uncertainty" in allowed: - for tail_focus_quantile in tail_focus_quantiles: - grid.append( - ( - "tail_blended_uncertainty", - float(gamma), - 1.0, - float(tail_focus_quantile), - ) - ) - if use_all or "segment_tail_blended_uncertainty" in allowed: - for tail_focus_quantile in tail_focus_quantiles: - grid.append( - ( - "segment_tail_blended_uncertainty", - float(gamma), - 1.0, - float(tail_focus_quantile), - ) - ) - if use_all or "segment_relative_tail_blended_uncertainty" in allowed: - for tail_focus_quantile in tail_focus_quantiles: - grid.append( - ( - "segment_relative_tail_blended_uncertainty", - float(gamma), - 1.0, - float(tail_focus_quantile), - ) - ) + allowed = _normalized_policy_mode_filter(policy_modes) + mode_order = [ + "blended_uncertainty", + "capped_blended_uncertainty", + "tail_blended_uncertainty", + "segment_tail_blended_uncertainty", + "segment_relative_tail_blended_uncertainty", + ] + active_modes = [mode for mode in mode_order if not allowed or mode in allowed] + grid = [ + (mode, float(gamma), delta_cap_quantile, tail_focus_quantile) + for gamma in gamma_values + for mode in active_modes + for delta_cap_quantile, tail_focus_quantile in _mode_quantile_pairs( + mode=mode, + delta_cap_quantiles=delta_cap_quantiles, + tail_focus_quantiles=tail_focus_quantiles, + ) + ] return list(dict.fromkeys(grid)) @@ -748,7 +831,12 @@ def _aggregate_frontier(frontier_raw: pd.DataFrame) -> pd.DataFrame: def _aggregate_alpha_grid_results(bound_eval: pd.DataFrame) -> pd.DataFrame: """Summarize exact bound checks across every evaluated alpha level.""" if bound_eval.empty: - return pd.DataFrame(columns=[*SEMANTIC_POLICY_FIELDS, "alpha_exact_pass_count"]) + return pd.DataFrame( + { + **{field: pd.Series(dtype="object") for field in SEMANTIC_POLICY_FIELDS}, + "alpha_exact_pass_count": pd.Series(dtype="int64"), + } + ) grouped = bound_eval.groupby(SEMANTIC_POLICY_FIELDS, dropna=False) out = grouped.agg( alpha_exact_pass_count=("all_bounds_hold", "sum"), @@ -1125,7 +1213,8 @@ def _bucket_payload(frame: pd.DataFrame) -> dict[str, Any]: exact_alpha_summary = {} if not bound_eval.empty: for alpha, frame in bound_eval.groupby("alpha", dropna=False): - exact_alpha_summary[str(float(alpha))] = { + alpha_value = float(str(alpha)) + exact_alpha_summary[str(alpha_value)] = { "n_checks": int(len(frame)), "pass_rate": float(frame["all_bounds_hold"].fillna(False).mean()), "max_violation": float(frame["violation"].max()), @@ -1157,7 +1246,7 @@ def _selection_reason(row: pd.Series) -> str: return "selected_best_available_without_alpha01_pass" -def main(argv: list[str] | None = None) -> int: +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument("--config", default="configs/crpto_optimization.yaml") parser.add_argument("--conformal-intervals-path", required=True) @@ -1175,7 +1264,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--bucket-proxy-k", type=int, default=40) parser.add_argument("--bucket-family-k", type=int, default=20) parser.add_argument("--bucket-region-k", type=int, default=20) - parser.add_argument("--alpha-grid", default="0.01,0.03,0.10") + parser.add_argument("--alpha-grid", default=IJDS_DECLARED_ALPHA_GRID_CSV) parser.add_argument("--max-candidates", type=int, default=5000) parser.add_argument( "--exact-max-candidates", @@ -1253,9 +1342,14 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument("--budget", type=float, default=1_000_000.0) parser.add_argument("--t-eval", type=float, default=0.05) - args = parser.parse_args(argv) + return parser + - run_label = str(args.run_label).strip().replace("/", "_") +def _sanitize_run_label(raw: object) -> str: + return str(raw).strip().replace("/", "_") + + +def _resolve_run_paths(args: argparse.Namespace, *, run_label: str) -> BoundAwarePaths: output_dir = ( Path(str(args.output_dir)).expanduser() if str(args.output_dir).strip() @@ -1266,44 +1360,33 @@ def main(argv: list[str] | None = None) -> int: if str(args.model_dir).strip() else ROOT / "models" / "portfolio_bound_aware" / run_label ) - output_dir.mkdir(parents=True, exist_ok=True) - model_dir.mkdir(parents=True, exist_ok=True) - - status_path = model_dir / f"{STAGE_NAME}_runtime_status.json" - checkpoint_dir = model_dir / f"{STAGE_NAME}_runtime_checkpoints" - resource_path = model_dir / "resource_snapshot.json" - gpu_csv_path = model_dir / "gpu_samples.csv" - - risk_values = _coerce_csv(args.risk_grid) - aversion_values = _coerce_csv(args.aversion_grid) - gamma_values = _coerce_csv(args.gamma_grid) - delta_cap_quantiles = _coerce_csv(args.delta_cap_grid) - tail_focus_quantiles = _coerce_csv(args.tail_focus_grid) - alpha_grid = _coerce_csv(args.alpha_grid) - random_states = _coerce_int_csv(args.random_states, fallback=int(args.random_state)) - exact_random_states = ( - _coerce_int_csv(args.exact_random_states, fallback=int(args.random_state)) - if str(args.exact_random_states).strip() - else list(random_states) - ) - exact_max_candidates = ( - int(args.exact_max_candidates) - if args.exact_max_candidates is not None - else int(args.max_candidates) + return BoundAwarePaths( + output_dir=output_dir, + model_dir=model_dir, + status_path=model_dir / f"{STAGE_NAME}_runtime_status.json", + checkpoint_dir=model_dir / f"{STAGE_NAME}_runtime_checkpoints", + resource_path=model_dir / "resource_snapshot.json", + gpu_csv_path=model_dir / "gpu_samples.csv", + frontier_raw_path=output_dir / "portfolio_bound_aware_frontier_raw.parquet", + frontier_path=output_dir / "portfolio_bound_aware_frontier.parquet", + shortlist_path=output_dir / "portfolio_bound_aware_shortlist.parquet", + shortlist_exact_path=output_dir / "portfolio_bound_aware_shortlist_exact.parquet", + bound_eval_path=output_dir / "portfolio_bound_aware_bound_eval.parquet", + region_summary_path=model_dir / "portfolio_bound_aware_region_summary.json", + selection_path=model_dir / "portfolio_bound_aware_selection.json", + exact_context_path=model_dir / "portfolio_bound_aware_exact_context.json", ) - incumbent_risk_neighbors = _coerce_csv(args.incumbent_risk_neighbors) - incumbent_gamma_neighbors = _coerce_csv(args.incumbent_gamma_neighbors) - incumbent_policy_modes = [ - part.strip() for part in str(args.incumbent_policy_modes).split(",") if part.strip() - ] - policy_modes = [part.strip() for part in str(args.policy_modes).split(",") if part.strip()] - exact_python_executable = str(args.exact_python_executable).strip() - exact_helper_script = Path(str(args.exact_helper_script)).resolve() + +def _ensure_run_dirs(paths: BoundAwarePaths) -> None: + paths.output_dir.mkdir(parents=True, exist_ok=True) + paths.model_dir.mkdir(parents=True, exist_ok=True) + + +def _budget_profiles(raw: object) -> list[dict[str, Any]]: budget_profiles: list[dict[str, Any]] = [] - for token in [ - part.strip().lower() for part in str(args.budget_profiles).split(",") if part.strip() - ]: + tokens = [part.strip().lower() for part in str(raw).split(",") if part.strip()] + for token in tokens: if token == "free": budget_profiles.append( {"name": "free_budget", "min_budget_utilization": 0.0, "pd_cap_slack_penalty": 0.0} @@ -1320,413 +1403,695 @@ def main(argv: list[str] | None = None) -> int: raise ValueError(f"Unsupported budget profile: {token}") if not budget_profiles: raise ValueError("At least one budget profile is required") + return budget_profiles - incumbent_policy = _load_incumbent_policy(args.incumbent_policy_path) - backend_validation: dict[str, Any] | None = None - if ( - str(args.solver_backend).strip().lower() == "cuopt" - or str(args.exact_solver_backend).strip().lower() == "cuopt" - ): - backend_validation = _validate_cuopt_runtime() + +def _search_space_payload( + *, + args: argparse.Namespace, + risk_values: list[float], + aversion_values: list[float], + gamma_values: list[float], + delta_cap_quantiles: list[float], + tail_focus_quantiles: list[float], + budget_profiles: list[dict[str, Any]], + alpha_grid: list[float], + random_states: list[int], + exact_random_states: list[int], + exact_max_candidates: int, + policy_modes: list[str], + cuopt_parameters: dict[str, Any], + incumbent_risk_neighbors: list[float], + incumbent_gamma_neighbors: list[float], + incumbent_policy_modes: list[str], +) -> dict[str, Any]: + return { + "risk_grid": risk_values, + "aversion_grid": aversion_values, + "gamma_grid": gamma_values, + "delta_cap_grid": delta_cap_quantiles, + "tail_focus_grid": tail_focus_quantiles, + "budget_profiles": budget_profiles, + "alpha_grid": alpha_grid, + "max_candidates": int(args.max_candidates), + "exact_max_candidates": int(exact_max_candidates), + "random_states": random_states, + "exact_random_states": exact_random_states, + "exact_checkpoint_every": int(args.exact_checkpoint_every), + "exact_threads": int(args.exact_threads), + "policy_modes": policy_modes, + "cuopt_parameters": cuopt_parameters, + "bucket_return_k": int(args.bucket_return_k), + "bucket_proxy_k": int(args.bucket_proxy_k), + "bucket_family_k": int(args.bucket_family_k), + "bucket_region_k": int(args.bucket_region_k), + "incumbent_policy_path": str(args.incumbent_policy_path), + "incumbent_risk_neighbors": incumbent_risk_neighbors, + "incumbent_gamma_neighbors": incumbent_gamma_neighbors, + "incumbent_policy_modes": incumbent_policy_modes, + } + + +def _selection_policy_payload() -> dict[str, Any]: + return { + "shortlist_strategy": "stratified_bound_first", + "rank_order": [ + "alpha01_exact_pass(desc)", + "alpha03_exact_pass(desc)", + "alpha_exact_pass_count(desc)", + "alpha_exact_pass_rate(desc)", + "ab_pass_all(desc)", + "realized_total_return(desc)", + "alpha01_weighted_miscoverage_V(asc)", + "alpha01_gamma_cp(asc)", + "price_of_robustness(asc)", + ], + } + + +def _selection_context_payload( + *, + args: argparse.Namespace, + paths: BoundAwarePaths, + run_label: str, + search_space: dict[str, Any], + exact_max_candidates: int, + random_states: list[int], + exact_random_states: list[int], + alpha_grid: list[float], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "run_label": run_label, + "conformal_intervals_path": str(args.conformal_intervals_path), + "search_space": search_space, + "selection_policy": _selection_policy_payload(), + "frontier_raw_path": str(paths.frontier_raw_path), + "frontier_path": str(paths.frontier_path), + "shortlist_path": str(paths.shortlist_path), + "shortlist_exact_path": str(paths.shortlist_exact_path), + "bound_eval_path": str(paths.bound_eval_path), + "region_summary_path": str(paths.region_summary_path), + "selection_path": str(paths.selection_path), + "runtime_status_path": str(paths.status_path), + "runtime_checkpoint_dir": str(paths.checkpoint_dir), + "resource_snapshot_path": str(paths.resource_path), + "frontier_solver_backend": str(args.solver_backend), + "exact_solver_backend": str(args.exact_solver_backend), + "budget": float(args.budget), + "t_eval": float(args.t_eval), + "max_candidates": int(args.max_candidates), + "exact_max_candidates": int(exact_max_candidates), + "random_states": random_states, + "exact_random_states": exact_random_states, + "exact_checkpoint_every": int(args.exact_checkpoint_every), + "exact_threads": int(args.exact_threads), + "alpha_grid": alpha_grid, + } + + +def _run_in_process_exact_bound_eval( + *, + args: argparse.Namespace, + shortlist: pd.DataFrame, + alpha_grid: list[float], + exact_random_states: list[int], + exact_max_candidates: int, + tracker: _ProgressTracker, +) -> pd.DataFrame: + aligned_by_seed = { + int(seed): _load_aligned_dataset( + conformal_intervals_path=args.conformal_intervals_path, + max_candidates=int(exact_max_candidates), + random_state=int(seed), + ) + for seed in exact_random_states + } + bound_rows: list[dict[str, Any]] = [] + completed_checks = 0 + for _, row in shortlist.iterrows(): + policy = _policy_from_row( + row, + solver_backend_override=str(args.exact_solver_backend), + ) + candidate_payload = row.to_dict() + for eval_seed in exact_random_states: + aligned = aligned_by_seed[int(eval_seed)] + for alpha in alpha_grid: + result = _validate_single_alpha( + aligned, + alpha=float(alpha), + policy=policy, + allocator_mode="exact", + budget=float(args.budget), + t_eval=float(args.t_eval), + threads=int(args.exact_threads), + ) + bound_rows.append( + { + "candidate_rank": int(candidate_payload["candidate_rank"]), + "eval_random_state": int(eval_seed), + "frontier_solver_backend": str(args.solver_backend), + "exact_solver_backend": str(args.exact_solver_backend), + **candidate_payload, + **result, + } + ) + completed_checks += 1 + tracker.bound_progress( + completed_checks=completed_checks, + extra={ + "candidate_rank": int(candidate_payload["candidate_rank"]), + "eval_random_state": int(eval_seed), + "current_alpha": float(alpha), + "exact_threads": int(args.exact_threads), + }, + ) + return pd.DataFrame(bound_rows) + + +def _build_selection_payload( + *, + args: argparse.Namespace, + run_label: str, + selection_context: dict[str, Any], + selected: pd.Series, + selected_policy: dict[str, Any], + region_payload: dict[str, Any], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "run_label": run_label, + "conformal_intervals_path": str(args.conformal_intervals_path), + "search_space": selection_context["search_space"], + "selection_policy": selection_context["selection_policy"], + "selected_policy": selected_policy, + "selected_metrics": selected.to_dict(), + "selection_reason": _selection_reason(selected), + "frontier_raw_path": selection_context["frontier_raw_path"], + "frontier_path": selection_context["frontier_path"], + "shortlist_path": selection_context["shortlist_path"], + "shortlist_exact_path": selection_context["shortlist_exact_path"], + "bound_eval_path": selection_context["bound_eval_path"], + "region_summary_path": selection_context["region_summary_path"], + "robust_region_summary": region_payload, + "runtime_status_path": selection_context["runtime_status_path"], + "runtime_checkpoint_dir": selection_context["runtime_checkpoint_dir"], + "resource_snapshot_path": selection_context["resource_snapshot_path"], + "frontier_solver_backend": str(args.solver_backend), + "exact_solver_backend": str(args.exact_solver_backend), + "exact_threads": int(args.exact_threads), + } + + +def _write_selection_outputs( + *, + paths: BoundAwarePaths, + shortlist_eval: pd.DataFrame, + bound_eval: pd.DataFrame, + region_payload: dict[str, Any], + payload: dict[str, Any], +) -> None: + atomic_write_parquet(shortlist_eval, paths.shortlist_exact_path, index=False) + atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) + atomic_write_json(paths.region_summary_path, region_payload) + atomic_write_json(paths.selection_path, payload) + + +def _build_frontier_outputs( + *, + args: argparse.Namespace, + risk_values: list[float], + aversion_values: list[float], + gamma_values: list[float], + delta_cap_quantiles: list[float], + tail_focus_quantiles: list[float], + budget_profiles: list[dict[str, Any]], + random_states: list[int], + policy_grid_count: int, + policy_modes: list[str], + cuopt_parameters: dict[str, Any], + tracker: _ProgressTracker, +) -> tuple[pd.DataFrame, pd.DataFrame]: + frontier_frames: list[pd.DataFrame] = [] + frontier_completed = 0 + for seed in random_states: + seed_offset = frontier_completed + + def _progress_hook( + local_completed: int, + extra: dict[str, Any], + _offset: int = seed_offset, + ) -> None: + tracker.frontier_progress(completed_units=_offset + local_completed, extra=extra) + + frontier_seed = _build_frontier_for_seed( + config_path=args.config, + conformal_intervals_path=args.conformal_intervals_path, + risk_values=risk_values, + aversion_values=aversion_values, + gamma_values=gamma_values, + delta_cap_quantiles=delta_cap_quantiles, + tail_focus_quantiles=tail_focus_quantiles, + budget_profiles=budget_profiles, + max_candidates=int(args.max_candidates), + random_state=int(seed), + solver_backend=str(args.solver_backend), + cuopt_presolve=int(args.cuopt_presolve) + if str(args.solver_backend) == "cuopt" + else None, + cuopt_parameters=cuopt_parameters, + policy_modes=policy_modes, + progress_hook=_progress_hook, + ) + frontier_frames.append(frontier_seed) + frontier_completed += int(len(risk_values) * (1 + policy_grid_count)) + + frontier_raw = ( + pd.concat(frontier_frames, ignore_index=True) if frontier_frames else pd.DataFrame() + ) + if frontier_raw.empty: + raise ValueError("Frontier search produced zero candidate rows.") + return frontier_raw, _aggregate_frontier(frontier_raw) + + +def _write_frontier_artifacts( + *, + paths: BoundAwarePaths, + frontier_raw: pd.DataFrame, + frontier: pd.DataFrame, + shortlist: pd.DataFrame, +) -> None: + atomic_write_parquet(frontier_raw, paths.frontier_raw_path, index=False) + atomic_write_parquet(frontier, paths.frontier_path, index=False) + atomic_write_parquet(shortlist, paths.shortlist_path, index=False) + + +def _complete_frontier_only( + *, + paths: BoundAwarePaths, + tracker: _ProgressTracker, + resource_payload: dict[str, Any], + shortlist_extra: dict[str, Any], + bound_total_checks: int, +) -> None: + resource_payload["end"] = _resource_snapshot() + atomic_write_json(paths.resource_path, resource_payload) + tracker.frontier_only_complete( + extra={ + **shortlist_extra, + "frontier_only": True, + "deferred_bound_total_checks": bound_total_checks, + "exact_context_path": str(paths.exact_context_path), + "frontier_path": str(paths.frontier_path), + "shortlist_path": str(paths.shortlist_path), + } + ) + + +def _delegate_exact_stage_if_requested( + *, + exact_python_executable: str, + exact_helper_script: Path, + paths: BoundAwarePaths, + resource_payload: dict[str, Any], +) -> bool: + if not exact_python_executable: + return False + current_python = Path(sys.executable) + requested_python = Path(exact_python_executable) + if requested_python == current_python: + return False + cmd = [ + str(requested_python), + str(exact_helper_script), + "--context-path", + str(paths.exact_context_path), + ] + logger.info("Delegating exact bound stage to external Python: {}", " ".join(cmd)) + subprocess.run(cmd, cwd=str(ROOT), check=True) + resource_payload["end"] = _resource_snapshot() + atomic_write_json(paths.resource_path, resource_payload) + return True + + +def _build_grid_spec(args: argparse.Namespace) -> BoundAwareGridSpec: + random_states = _coerce_int_csv(args.random_states, fallback=int(args.random_state)) + exact_random_states = ( + _coerce_int_csv(args.exact_random_states, fallback=int(args.random_state)) + if str(args.exact_random_states).strip() + else list(random_states) + ) + exact_max_candidates = ( + int(args.exact_max_candidates) + if args.exact_max_candidates is not None + else int(args.max_candidates) + ) + return BoundAwareGridSpec( + risk_values=_coerce_csv(args.risk_grid), + aversion_values=_coerce_csv(args.aversion_grid), + gamma_values=_coerce_csv(args.gamma_grid), + delta_cap_quantiles=_coerce_csv(args.delta_cap_grid), + tail_focus_quantiles=_coerce_csv(args.tail_focus_grid), + alpha_grid=_coerce_csv(args.alpha_grid), + random_states=random_states, + exact_random_states=exact_random_states, + exact_max_candidates=exact_max_candidates, + budget_profiles=_budget_profiles(args.budget_profiles), + policy_modes=_coerce_str_csv(args.policy_modes), + incumbent_risk_neighbors=_coerce_csv(args.incumbent_risk_neighbors), + incumbent_gamma_neighbors=_coerce_csv(args.incumbent_gamma_neighbors), + incumbent_policy_modes=_coerce_str_csv(args.incumbent_policy_modes), + ) + + +def _prepare_execution_spec(args: argparse.Namespace) -> BoundAwareExecutionSpec: + run_label = _sanitize_run_label(args.run_label) + paths = _resolve_run_paths(args, run_label=run_label) + _ensure_run_dirs(paths) + grid = _build_grid_spec(args) + uses_cuopt = "cuopt" in { + str(args.solver_backend).strip().lower(), + str(args.exact_solver_backend).strip().lower(), + } + backend_validation = _validate_cuopt_runtime() if uses_cuopt else None cuopt_parameters = ( - _cuopt_parameter_overrides(args, model_dir=model_dir) + _cuopt_parameter_overrides(args, model_dir=paths.model_dir) if str(args.solver_backend).strip().lower() == "cuopt" else {} ) - - policy_grid_count = _policy_grid_size( - gamma_values=gamma_values, - delta_cap_quantiles=delta_cap_quantiles, - tail_focus_quantiles=tail_focus_quantiles, - aversion_values=aversion_values, - budget_profiles=budget_profiles, - policy_modes=policy_modes, + return BoundAwareExecutionSpec( + run_label=run_label, + paths=paths, + grid=grid, + incumbent_policy=_load_incumbent_policy(args.incumbent_policy_path), + exact_python_executable=str(args.exact_python_executable).strip(), + exact_helper_script=Path(str(args.exact_helper_script)).resolve(), + backend_validation=backend_validation, + cuopt_parameters=cuopt_parameters, ) - frontier_total_units = int(len(random_states) * len(risk_values) * (1 + policy_grid_count)) + + +def _initialize_run_context(args: argparse.Namespace) -> BoundAwareRunContext: + spec = _prepare_execution_spec(args) + grid = spec.grid tracker = _ProgressTracker( - status_path=status_path, - checkpoint_dir=checkpoint_dir, - run_tag=run_label, - frontier_total_units=frontier_total_units, + status_path=spec.paths.status_path, + checkpoint_dir=spec.paths.checkpoint_dir, + run_tag=spec.run_label, + frontier_total_units=grid.frontier_total_units, + ) + search_space = _search_space_payload( + args=args, + risk_values=grid.risk_values, + aversion_values=grid.aversion_values, + gamma_values=grid.gamma_values, + delta_cap_quantiles=grid.delta_cap_quantiles, + tail_focus_quantiles=grid.tail_focus_quantiles, + budget_profiles=grid.budget_profiles, + alpha_grid=grid.alpha_grid, + random_states=grid.random_states, + exact_random_states=grid.exact_random_states, + exact_max_candidates=grid.exact_max_candidates, + policy_modes=grid.policy_modes, + cuopt_parameters=spec.cuopt_parameters, + incumbent_risk_neighbors=grid.incumbent_risk_neighbors, + incumbent_gamma_neighbors=grid.incumbent_gamma_neighbors, + incumbent_policy_modes=grid.incumbent_policy_modes, ) resource_payload = { "schema_version": SCHEMA_VERSION, - "run_label": run_label, + "run_label": spec.run_label, "solver_backend": str(args.solver_backend), "exact_solver_backend": str(args.exact_solver_backend), - "cuopt_parameters": cuopt_parameters, + "cuopt_parameters": spec.cuopt_parameters, "start": _resource_snapshot(), - "backend_validation": backend_validation, + "backend_validation": spec.backend_validation, } - atomic_write_json(resource_path, resource_payload) + atomic_write_json(spec.paths.resource_path, resource_payload) tracker.start( extra={ - "random_states": random_states, - "search_space": { - "risk_grid": risk_values, - "aversion_grid": aversion_values, - "gamma_grid": gamma_values, - "delta_cap_grid": delta_cap_quantiles, - "tail_focus_grid": tail_focus_quantiles, - "budget_profiles": budget_profiles, - "alpha_grid": alpha_grid, - "max_candidates": int(args.max_candidates), - "exact_max_candidates": int(exact_max_candidates), - "exact_random_states": exact_random_states, - "exact_checkpoint_every": int(args.exact_checkpoint_every), - "exact_threads": int(args.exact_threads), - "cuopt_parameters": cuopt_parameters, - }, + "random_states": grid.random_states, + "search_space": search_space, } ) + return BoundAwareRunContext( + args=args, + spec=spec, + tracker=tracker, + search_space=search_space, + resource_payload=resource_payload, + ) - gpu_sampler: _GpuSampler | None = None - if str(args.solver_backend).strip().lower() == "cuopt": - gpu_sampler = _GpuSampler(gpu_csv_path) + +def _start_gpu_sampler(context: BoundAwareRunContext) -> _GpuSampler | None: + if str(context.args.solver_backend).strip().lower() == "cuopt": + gpu_sampler = _GpuSampler(context.spec.paths.gpu_csv_path) gpu_sampler.start() + return gpu_sampler + return None + + +def _build_frontier_state(context: BoundAwareRunContext) -> BoundAwareFrontierState: + args = context.args + spec = context.spec + grid = spec.grid + tracker = context.tracker + frontier_raw, frontier = _build_frontier_outputs( + args=args, + risk_values=grid.risk_values, + aversion_values=grid.aversion_values, + gamma_values=grid.gamma_values, + delta_cap_quantiles=grid.delta_cap_quantiles, + tail_focus_quantiles=grid.tail_focus_quantiles, + budget_profiles=grid.budget_profiles, + random_states=grid.random_states, + policy_grid_count=grid.policy_grid_count, + policy_modes=grid.policy_modes, + cuopt_parameters=spec.cuopt_parameters, + tracker=tracker, + ) + tracker.frontier_complete( + extra={ + "frontier_policy_count": len(frontier), + "frontier_raw_rows": len(frontier_raw), + } + ) + shortlist = _build_stratified_shortlist( + frontier=frontier, + shortlist_top_k=int(args.shortlist_top_k), + bucket_return_k=int(args.bucket_return_k), + bucket_proxy_k=int(args.bucket_proxy_k), + bucket_family_k=int(args.bucket_family_k), + bucket_region_k=int(args.bucket_region_k), + incumbent_policy=spec.incumbent_policy, + incumbent_risk_neighbors=grid.incumbent_risk_neighbors, + incumbent_gamma_neighbors=grid.incumbent_gamma_neighbors, + incumbent_policy_modes=grid.incumbent_policy_modes, + budget_profiles=grid.budget_profiles, + solver_backend=str(args.solver_backend), + ) + bound_total_checks = grid.bound_total_checks(len(shortlist)) + shortlist_extra = { + "shortlist_size": len(shortlist), + "shortlist_buckets": shortlist["shortlist_bucket"].value_counts(dropna=False).to_dict(), + } + if not args.frontier_only: + tracker.set_bound_total(bound_total_checks, extra=shortlist_extra) + _write_frontier_artifacts( + paths=spec.paths, + frontier_raw=frontier_raw, + frontier=frontier, + shortlist=shortlist, + ) + selection_context = _selection_context_payload( + args=args, + paths=spec.paths, + run_label=spec.run_label, + search_space=context.search_space, + exact_max_candidates=grid.exact_max_candidates, + random_states=grid.random_states, + exact_random_states=grid.exact_random_states, + alpha_grid=grid.alpha_grid, + ) + atomic_write_json(spec.paths.exact_context_path, selection_context) + return BoundAwareFrontierState( + frontier_raw=frontier_raw, + frontier=frontier, + shortlist=shortlist, + bound_total_checks=bound_total_checks, + shortlist_extra=shortlist_extra, + selection_context=selection_context, + ) - frontier_frames: list[pd.DataFrame] = [] - frontier_completed = 0 - try: - for seed in random_states: - seed_offset = frontier_completed - - def _progress_hook( - local_completed: int, - extra: dict[str, Any], - _offset: int = seed_offset, - ) -> None: - tracker.frontier_progress(completed_units=_offset + local_completed, extra=extra) - - frontier_seed = _build_frontier_for_seed( - config_path=args.config, - conformal_intervals_path=args.conformal_intervals_path, - risk_values=risk_values, - aversion_values=aversion_values, - gamma_values=gamma_values, - delta_cap_quantiles=delta_cap_quantiles, - tail_focus_quantiles=tail_focus_quantiles, - budget_profiles=budget_profiles, - max_candidates=int(args.max_candidates), - random_state=int(seed), - solver_backend=str(args.solver_backend), - cuopt_presolve=int(args.cuopt_presolve) - if str(args.solver_backend) == "cuopt" - else None, - cuopt_parameters=cuopt_parameters, - policy_modes=policy_modes, - progress_hook=_progress_hook, - ) - frontier_frames.append(frontier_seed) - frontier_completed += int(len(risk_values) * (1 + policy_grid_count)) - frontier_raw = ( - pd.concat(frontier_frames, ignore_index=True) if frontier_frames else pd.DataFrame() - ) - if frontier_raw.empty: - raise ValueError("Frontier search produced zero candidate rows.") - frontier = _aggregate_frontier(frontier_raw) - tracker.frontier_complete( - extra={ - "frontier_policy_count": len(frontier), - "frontier_raw_rows": len(frontier_raw), - } +def _finish_after_frontier( + context: BoundAwareRunContext, + state: BoundAwareFrontierState, +) -> bool: + spec = context.spec + if context.args.frontier_only: + _complete_frontier_only( + paths=spec.paths, + tracker=context.tracker, + resource_payload=context.resource_payload, + shortlist_extra=state.shortlist_extra, + bound_total_checks=state.bound_total_checks, ) + return True + return _delegate_exact_stage_if_requested( + exact_python_executable=spec.exact_python_executable, + exact_helper_script=spec.exact_helper_script, + paths=spec.paths, + resource_payload=context.resource_payload, + ) - shortlist = _build_stratified_shortlist( - frontier=frontier, - shortlist_top_k=int(args.shortlist_top_k), - bucket_return_k=int(args.bucket_return_k), - bucket_proxy_k=int(args.bucket_proxy_k), - bucket_family_k=int(args.bucket_family_k), - bucket_region_k=int(args.bucket_region_k), - incumbent_policy=incumbent_policy, - incumbent_risk_neighbors=incumbent_risk_neighbors, - incumbent_gamma_neighbors=incumbent_gamma_neighbors, - incumbent_policy_modes=incumbent_policy_modes, - budget_profiles=budget_profiles, - solver_backend=str(args.solver_backend), - ) - bound_total_checks = int(len(shortlist) * len(alpha_grid) * len(exact_random_states)) - shortlist_extra = { - "shortlist_size": len(shortlist), - "shortlist_buckets": shortlist["shortlist_bucket"].value_counts(dropna=False).to_dict(), - } - if not args.frontier_only: - tracker.set_bound_total(bound_total_checks, extra=shortlist_extra) - atomic_write_parquet( - frontier_raw, output_dir / "portfolio_bound_aware_frontier_raw.parquet", index=False - ) - atomic_write_parquet( - frontier, output_dir / "portfolio_bound_aware_frontier.parquet", index=False - ) - atomic_write_parquet( - shortlist, output_dir / "portfolio_bound_aware_shortlist.parquet", index=False - ) +def _run_exact_selection( + context: BoundAwareRunContext, + state: BoundAwareFrontierState, +) -> tuple[pd.Series, dict[str, Any]]: + args = context.args + spec = context.spec + grid = spec.grid + bound_eval = _run_in_process_exact_bound_eval( + args=args, + shortlist=state.shortlist, + alpha_grid=grid.alpha_grid, + exact_random_states=grid.exact_random_states, + exact_max_candidates=grid.exact_max_candidates, + tracker=context.tracker, + ) + shortlist_eval = _aggregate_exact_results(shortlist=state.shortlist, bound_eval=bound_eval) + region_payload = _region_summary(shortlist_eval, bound_eval) + selected = shortlist_eval.iloc[0].copy() + selected_policy = _policy_from_row( + selected, + solver_backend_override=str(args.exact_solver_backend), + ) + payload = _build_selection_payload( + args=args, + run_label=spec.run_label, + selection_context=state.selection_context, + selected=selected, + selected_policy=selected_policy, + region_payload=region_payload, + ) + _write_selection_outputs( + paths=spec.paths, + shortlist_eval=shortlist_eval, + bound_eval=bound_eval, + region_payload=region_payload, + payload=payload, + ) + return selected, payload - selection_context = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_label": run_label, - "conformal_intervals_path": str(args.conformal_intervals_path), - "search_space": { - "risk_grid": risk_values, - "aversion_grid": aversion_values, - "gamma_grid": gamma_values, - "delta_cap_grid": delta_cap_quantiles, - "tail_focus_grid": tail_focus_quantiles, - "budget_profiles": budget_profiles, - "alpha_grid": alpha_grid, - "max_candidates": int(args.max_candidates), - "exact_max_candidates": int(exact_max_candidates), - "random_states": random_states, - "exact_random_states": exact_random_states, - "exact_checkpoint_every": int(args.exact_checkpoint_every), - "exact_threads": int(args.exact_threads), - "policy_modes": policy_modes, - "cuopt_parameters": cuopt_parameters, - "bucket_return_k": int(args.bucket_return_k), - "bucket_proxy_k": int(args.bucket_proxy_k), - "bucket_family_k": int(args.bucket_family_k), - "bucket_region_k": int(args.bucket_region_k), - "incumbent_policy_path": str(args.incumbent_policy_path), - "incumbent_risk_neighbors": incumbent_risk_neighbors, - "incumbent_gamma_neighbors": incumbent_gamma_neighbors, - "incumbent_policy_modes": incumbent_policy_modes, - }, - "selection_policy": { - "shortlist_strategy": "stratified_bound_first", - "rank_order": [ - "alpha01_exact_pass(desc)", - "alpha03_exact_pass(desc)", - "alpha_exact_pass_count(desc)", - "alpha_exact_pass_rate(desc)", - "ab_pass_all(desc)", - "realized_total_return(desc)", - "alpha01_weighted_miscoverage_V(asc)", - "alpha01_gamma_cp(asc)", - "price_of_robustness(asc)", - ], - }, - "frontier_raw_path": str(output_dir / "portfolio_bound_aware_frontier_raw.parquet"), - "frontier_path": str(output_dir / "portfolio_bound_aware_frontier.parquet"), - "shortlist_path": str(output_dir / "portfolio_bound_aware_shortlist.parquet"), - "shortlist_exact_path": str( - output_dir / "portfolio_bound_aware_shortlist_exact.parquet" - ), - "bound_eval_path": str(output_dir / "portfolio_bound_aware_bound_eval.parquet"), - "region_summary_path": str(model_dir / "portfolio_bound_aware_region_summary.json"), - "selection_path": str(model_dir / "portfolio_bound_aware_selection.json"), - "runtime_status_path": str(status_path), - "runtime_checkpoint_dir": str(checkpoint_dir), - "resource_snapshot_path": str(resource_path), - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - "budget": float(args.budget), - "t_eval": float(args.t_eval), - "max_candidates": int(args.max_candidates), - "exact_max_candidates": int(exact_max_candidates), - "random_states": random_states, - "exact_random_states": exact_random_states, - "exact_checkpoint_every": int(args.exact_checkpoint_every), - "exact_threads": int(args.exact_threads), - "alpha_grid": alpha_grid, - } - exact_context_path = model_dir / "portfolio_bound_aware_exact_context.json" - atomic_write_json(exact_context_path, selection_context) - - if args.frontier_only: - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - tracker.frontier_only_complete( - extra={ - **shortlist_extra, - "frontier_only": True, - "deferred_bound_total_checks": bound_total_checks, - "exact_context_path": str(exact_context_path), - "frontier_path": str(output_dir / "portfolio_bound_aware_frontier.parquet"), - "shortlist_path": str(output_dir / "portfolio_bound_aware_shortlist.parquet"), - } - ) - return 0 - - if exact_python_executable: - current_python = Path(sys.executable) - requested_python = Path(exact_python_executable) - if requested_python != current_python: - cmd = [ - str(requested_python), - str(exact_helper_script), - "--context-path", - str(exact_context_path), - ] - logger.info( - "Delegating exact bound stage to external Python: {}", - " ".join(cmd), - ) - subprocess.run(cmd, cwd=str(ROOT), check=True) - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - return 0 - - aligned_by_seed = { - int(seed): _load_aligned_dataset( - conformal_intervals_path=args.conformal_intervals_path, - max_candidates=int(exact_max_candidates), - random_state=int(seed), - ) - for seed in exact_random_states - } - bound_rows: list[dict[str, Any]] = [] - completed_checks = 0 - for _, row in shortlist.iterrows(): - policy = _policy_from_row( - row, - solver_backend_override=str(args.exact_solver_backend), - ) - candidate_payload = row.to_dict() - for eval_seed in exact_random_states: - aligned = aligned_by_seed[int(eval_seed)] - for alpha in alpha_grid: - result = _validate_single_alpha( - aligned, - alpha=float(alpha), - policy=policy, - allocator_mode="exact", - budget=float(args.budget), - t_eval=float(args.t_eval), - threads=int(args.exact_threads), - ) - bound_rows.append( - { - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - **candidate_payload, - **result, - } - ) - completed_checks += 1 - tracker.bound_progress( - completed_checks=completed_checks, - extra={ - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "current_alpha": float(alpha), - "exact_threads": int(args.exact_threads), - }, - ) - bound_eval = pd.DataFrame(bound_rows) - shortlist_eval = _aggregate_exact_results(shortlist=shortlist, bound_eval=bound_eval) - region_payload = _region_summary(shortlist_eval, bound_eval) - selected = shortlist_eval.iloc[0].copy() - selected_policy = _policy_from_row( - selected, - solver_backend_override=str(args.exact_solver_backend), - ) - payload = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_label": run_label, - "conformal_intervals_path": str(args.conformal_intervals_path), - "search_space": selection_context["search_space"], - "selection_policy": selection_context["selection_policy"], - "selected_policy": selected_policy, - "selected_metrics": selected.to_dict(), - "selection_reason": _selection_reason(selected), - "frontier_raw_path": selection_context["frontier_raw_path"], - "frontier_path": selection_context["frontier_path"], - "shortlist_path": selection_context["shortlist_path"], - "shortlist_exact_path": selection_context["shortlist_exact_path"], - "bound_eval_path": selection_context["bound_eval_path"], - "region_summary_path": selection_context["region_summary_path"], - "robust_region_summary": region_payload, - "runtime_status_path": selection_context["runtime_status_path"], - "runtime_checkpoint_dir": selection_context["runtime_checkpoint_dir"], - "resource_snapshot_path": selection_context["resource_snapshot_path"], - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - "exact_threads": int(args.exact_threads), +def _complete_run( + context: BoundAwareRunContext, + *, + selected: pd.Series, + payload: dict[str, Any], + gpu_sampler: _GpuSampler | None, +) -> None: + if gpu_sampler is not None: + context.resource_payload["gpu_summary"] = gpu_sampler.stop() + context.resource_payload["end"] = _resource_snapshot() + atomic_write_json(context.spec.paths.resource_path, context.resource_payload) + context.tracker.complete( + extra={ + "selection_reason": str(payload["selection_reason"]), + "selected_alpha01_exact_pass": bool(selected["alpha01_exact_pass"]), + "selected_realized_total_return": float(selected["realized_total_return"]), } + ) - atomic_write_parquet( - shortlist_eval, - output_dir / "portfolio_bound_aware_shortlist_exact.parquet", - index=False, - ) - atomic_write_parquet( - bound_eval, output_dir / "portfolio_bound_aware_bound_eval.parquet", index=False - ) - atomic_write_json(model_dir / "portfolio_bound_aware_region_summary.json", region_payload) - atomic_write_json(model_dir / "portfolio_bound_aware_selection.json", payload) - - if gpu_sampler is not None: - resource_payload["gpu_summary"] = gpu_sampler.stop() - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - tracker.complete( - extra={ - "selection_reason": str(payload["selection_reason"]), - "selected_alpha01_exact_pass": bool(selected["alpha01_exact_pass"]), - "selected_realized_total_return": float(selected["realized_total_return"]), - } - ) - logger.info( - "Focused bound-aware search complete: selected risk_tolerance={}, mode={}, gamma={}, q_cap={}, q_tail={}, ab_pass_all={}, alpha01_pass={}", - selected["risk_tolerance"], - selected["policy_mode"], - selected["gamma"], - selected["delta_cap_quantile"], - selected["tail_focus_quantile"], - selected["ab_pass_all"], - selected["alpha01_exact_pass"], - ) - logger.info( - "Saved frontier raw: {}", output_dir / "portfolio_bound_aware_frontier_raw.parquet" - ) - logger.info( - "Saved frontier aggregate: {}", output_dir / "portfolio_bound_aware_frontier.parquet" - ) - logger.info("Saved shortlist: {}", output_dir / "portfolio_bound_aware_shortlist.parquet") - logger.info( - "Saved exact shortlist: {}", - output_dir / "portfolio_bound_aware_shortlist_exact.parquet", - ) - logger.info( - "Saved bound evaluations: {}", output_dir / "portfolio_bound_aware_bound_eval.parquet" - ) - logger.info( - "Saved selection payload: {}", model_dir / "portfolio_bound_aware_selection.json" - ) +def _log_completed_run( + paths: BoundAwarePaths, + *, + selected: pd.Series, +) -> None: + logger.info( + "Focused bound-aware search complete: selected risk_tolerance={}, mode={}, gamma={}, q_cap={}, q_tail={}, ab_pass_all={}, alpha01_pass={}", + selected["risk_tolerance"], + selected["policy_mode"], + selected["gamma"], + selected["delta_cap_quantile"], + selected["tail_focus_quantile"], + selected["ab_pass_all"], + selected["alpha01_exact_pass"], + ) + for label, path in ( + ("frontier raw", paths.frontier_raw_path), + ("frontier aggregate", paths.frontier_path), + ("shortlist", paths.shortlist_path), + ("exact shortlist", paths.shortlist_exact_path), + ("bound evaluations", paths.bound_eval_path), + ("selection payload", paths.selection_path), + ): + logger.info("Saved {}: {}", label, path) + + +def _execute_bound_aware_search( + context: BoundAwareRunContext, + *, + gpu_sampler: _GpuSampler | None, +) -> int: + state = _build_frontier_state(context) + if _finish_after_frontier(context, state): return 0 + selected, payload = _run_exact_selection(context, state) + _complete_run(context, selected=selected, payload=payload, gpu_sampler=gpu_sampler) + _log_completed_run(context.spec.paths, selected=selected) + return 0 + + +def _record_run_failure(context: BoundAwareRunContext, exc: Exception) -> None: + tracker = context.tracker + error_payload = { + "error_type": type(exc).__name__, + "error": str(exc), + "frontier_completed_units": int(tracker.frontier_completed_units), + "frontier_total_units": int(tracker.frontier_total_units), + "bound_completed_checks": int(tracker.bound_completed_checks), + "bound_total_checks": int(tracker.bound_total_checks), + } + context.resource_payload["error"] = error_payload + context.resource_payload["end"] = _resource_snapshot() + atomic_write_json(context.spec.paths.resource_path, context.resource_payload) + tracker.fail(phase="failed", extra=error_payload) + logger.exception("Focused bound-aware portfolio search failed.") + + +def _finalize_gpu_sampler( + context: BoundAwareRunContext, + gpu_sampler: _GpuSampler | None, +) -> None: + if gpu_sampler is None or "gpu_summary" in context.resource_payload: + return + try: + context.resource_payload["gpu_summary"] = gpu_sampler.stop() + context.resource_payload["end"] = _resource_snapshot() + atomic_write_json(context.spec.paths.resource_path, context.resource_payload) + except Exception: # pragma: no cover - best effort cleanup only + pass + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + context = _initialize_run_context(args) + gpu_sampler = _start_gpu_sampler(context) + + try: + return _execute_bound_aware_search(context, gpu_sampler=gpu_sampler) except Exception as exc: - error_payload = { - "error_type": type(exc).__name__, - "error": str(exc), - "frontier_completed_units": int(tracker.frontier_completed_units), - "frontier_total_units": int(tracker.frontier_total_units), - "bound_completed_checks": int(tracker.bound_completed_checks), - "bound_total_checks": int(tracker.bound_total_checks), - } - resource_payload["error"] = error_payload - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - tracker.fail(phase="failed", extra=error_payload) - logger.exception("Focused bound-aware portfolio search failed.") + _record_run_failure(context, exc) raise finally: - if gpu_sampler is not None: - try: - if "gpu_summary" not in resource_payload: - resource_payload["gpu_summary"] = gpu_sampler.stop() - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - except Exception: # pragma: no cover - best effort cleanup only - pass + _finalize_gpu_sampler(context, gpu_sampler) if __name__ == "__main__": diff --git a/scripts/search/run_portfolio_bound_exact_eval.py b/scripts/search/run_portfolio_bound_exact_eval.py index dfe4608..37dc7e2 100644 --- a/scripts/search/run_portfolio_bound_exact_eval.py +++ b/scripts/search/run_portfolio_bound_exact_eval.py @@ -7,6 +7,8 @@ import os import sys import time +from collections.abc import Iterable, Iterator +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -54,6 +56,38 @@ ) +@dataclass(frozen=True) +class ExactEvalPaths: + status_path: Path + checkpoint_dir: Path + resource_snapshot_path: Path + shortlist_path: Path + shortlist_exact_path: Path + bound_eval_path: Path + selection_path: Path + + +@dataclass(frozen=True) +class ExactEvalPlan: + alpha_grid: list[float] + exact_max_candidates: int + requested_random_states: list[int] + random_states: list[int] + full_universe_seed_deduped: bool + expected_checks: int + exact_threads: int + checkpoint_every: int + + +@dataclass(frozen=True) +class ExactEvalTask: + cache_key: tuple[int, int, float] + candidate_payload: dict[str, object] + policy: dict[str, object] + eval_random_state: int + alpha: float + + def _resolve_repo_path(raw_path: object) -> Path: path_text = str(raw_path) path = Path(path_text) @@ -80,6 +114,32 @@ def _normalize_context_paths(context: dict[str, object]) -> None: context[key] = str(_resolve_repo_path(context[key])) +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--context-path", required=True) + return parser + + +def _load_context(context_path: Path) -> dict[str, object]: + context = json.loads(context_path.read_text(encoding="utf-8")) + if not isinstance(context, dict): + raise TypeError("Exact bound context must be a JSON object.") + _normalize_context_paths(context) + return context + + +def _exact_eval_paths(context: dict[str, object]) -> ExactEvalPaths: + return ExactEvalPaths( + status_path=Path(str(context["runtime_status_path"])), + checkpoint_dir=Path(str(context["runtime_checkpoint_dir"])), + resource_snapshot_path=Path(str(context["resource_snapshot_path"])), + shortlist_path=Path(str(context["shortlist_path"])), + shortlist_exact_path=_shortlist_exact_path(context), + bound_eval_path=Path(str(context["bound_eval_path"])), + selection_path=Path(str(context["selection_path"])), + ) + + def _eta_seconds(elapsed_sec: float, completed: int, total: int) -> float | None: if completed <= 0 or total <= 0 or completed >= total: return 0.0 if total > 0 and completed >= total else None @@ -147,25 +207,85 @@ def _load_partial_bound_eval(*, bound_eval_path: Path) -> pd.DataFrame: def _context_random_states(context: dict[str, object]) -> list[int]: raw = context.get("exact_random_states", context["random_states"]) + values = _context_value_list(raw) + return [_context_int(value) for value in values] + + +def _context_value_list(raw: object) -> list[object]: if isinstance(raw, str): - values = [part.strip() for part in raw.split(",") if part.strip()] - else: - values = list(raw) # type: ignore[arg-type] - return [int(v) for v in values] + return [part.strip() for part in raw.split(",") if part.strip()] + if isinstance(raw, Iterable): + return list(raw) + raise TypeError(f"Expected scalar list or comma-separated string, got {type(raw).__name__}") + + +def _context_int(raw: object) -> int: + if isinstance(raw, str | int | float): + return int(raw) + raise TypeError(f"Expected int-like value, got {type(raw).__name__}") def _context_max_candidates(context: dict[str, object]) -> int: - return int(context.get("exact_max_candidates", context["max_candidates"])) + return _context_int(context.get("exact_max_candidates", context["max_candidates"])) def _context_exact_threads(context: dict[str, object]) -> int: raw = os.environ.get("EXACT_THREADS", context.get("exact_threads", DEFAULT_THREADS)) try: - return max(1, int(raw)) + return max(1, _context_int(raw)) except (TypeError, ValueError): return DEFAULT_THREADS +def _requested_random_states(context: dict[str, object]) -> list[int]: + raw = context.get("requested_exact_random_states") + if raw is None: + return _context_random_states(context) + return [_context_int(value) for value in _context_value_list(raw)] + + +def _dedupe_full_universe_random_states( + *, + random_states: list[int], + exact_max_candidates: int, +) -> tuple[list[int], bool]: + if exact_max_candidates > 0 or len(random_states) <= 1: + return list(random_states), False + effective = [int(random_states[0])] + logger.info( + "Deduplicating exact_random_states for full-universe exact rerank: requested={} " + "effective={}. With exact_max_candidates<=0 there is no sampling, so seeds are " + "identical.", + random_states, + effective, + ) + return effective, True + + +def _exact_eval_plan( + *, + context: dict[str, object], + shortlist_rows: int, +) -> ExactEvalPlan: + alpha_grid = [_as_float(value) for value in _context_value_list(context["alpha_grid"])] + exact_max_candidates = _context_max_candidates(context) + requested_random_states = _requested_random_states(context) + random_states, full_universe_seed_deduped = _dedupe_full_universe_random_states( + random_states=requested_random_states, + exact_max_candidates=exact_max_candidates, + ) + return ExactEvalPlan( + alpha_grid=alpha_grid, + exact_max_candidates=exact_max_candidates, + requested_random_states=requested_random_states, + random_states=random_states, + full_universe_seed_deduped=full_universe_seed_deduped, + expected_checks=int(shortlist_rows * len(random_states) * len(alpha_grid)), + exact_threads=_context_exact_threads(context), + checkpoint_every=max(1, _context_int(context.get("exact_checkpoint_every", 100))), + ) + + def _validate_alpha_grid_supported(alpha_grid: list[float]) -> None: sweep_path = ROOT / "data" / "processed" / "alpha_sweep_pareto_mondrian.parquet" if not sweep_path.exists(): @@ -209,7 +329,7 @@ def _write_exact_status( frontier_total = _as_int(context["frontier_total_units"]) frontier_completed = _as_int(context["frontier_completed_units"]) bound_total = _as_int(context["bound_total_checks"]) - resume_completed = int(context.get("resume_completed_checks", 0) or 0) + resume_completed = _context_int(context.get("resume_completed_checks", 0) or 0) exact_elapsed_sec = time.monotonic() - started elapsed_sec = base_elapsed_sec + exact_elapsed_sec global_total = frontier_total + bound_total @@ -258,22 +378,28 @@ def _write_exact_status( return payload -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--context-path", required=True) - args = parser.parse_args(argv) +def _plan_status_extra( + plan: ExactEvalPlan, + extra: dict[str, object] | None = None, +) -> dict[str, object]: + payload: dict[str, object] = { + "checkpoint_every": int(plan.checkpoint_every), + "exact_max_candidates": int(plan.exact_max_candidates), + "exact_random_states": plan.random_states, + "requested_exact_random_states": plan.requested_random_states, + "full_universe_seed_deduped": bool(plan.full_universe_seed_deduped), + "exact_threads": int(plan.exact_threads), + } + if extra: + payload.update(extra) + return payload - context_path = Path(args.context_path).resolve() - context = json.loads(context_path.read_text(encoding="utf-8")) - _normalize_context_paths(context) - status_path = Path(str(context["runtime_status_path"])) - checkpoint_dir = Path(str(context["runtime_checkpoint_dir"])) - resource_snapshot_path = Path(str(context["resource_snapshot_path"])) - shortlist_path = Path(str(context["shortlist_path"])) - shortlist_exact_path = _shortlist_exact_path(context) - bound_eval_path = Path(str(context["bound_eval_path"])) - selection_path = Path(str(context["selection_path"])) +def _initialize_exact_progress( + *, + context: dict[str, object], + status_path: Path, +) -> float: prior_status = load_runtime_status(status_path) base_elapsed_sec = ( float(prior_status.get("elapsed_sec", 0.0)) @@ -284,167 +410,224 @@ def main(argv: list[str] | None = None) -> int: context["frontier_completed_units"] = int(prior_status.get("frontier_completed_units", 0)) context["bound_total_checks"] = int(prior_status.get("bound_total_checks", 0)) context["helper_started_monotonic"] = time.monotonic() + return base_elapsed_sec - shortlist = pd.read_parquet(shortlist_path) - alpha_grid = [float(v) for v in context["alpha_grid"]] - _validate_alpha_grid_supported(alpha_grid) - exact_max_candidates = _context_max_candidates(context) - requested_random_states_raw = context.get("requested_exact_random_states") - if requested_random_states_raw is None: - requested_random_states = _context_random_states(context) - elif isinstance(requested_random_states_raw, str): - requested_random_states = [ - int(part.strip()) for part in requested_random_states_raw.split(",") if part.strip() - ] - else: - requested_random_states = [int(value) for value in requested_random_states_raw] # type: ignore[arg-type] - random_states = list(requested_random_states) - full_universe_seed_deduped = False - if exact_max_candidates <= 0 and len(random_states) > 1: - random_states = [int(random_states[0])] - full_universe_seed_deduped = True - logger.info( - "Deduplicating exact_random_states for full-universe exact rerank: requested={} " - "effective={}. With exact_max_candidates<=0 there is no sampling, so seeds are " - "identical.", - requested_random_states, - random_states, + +def _resume_exact_rows( + *, + bound_eval_path: Path, + random_states: list[int], +) -> tuple[list[dict[str, object]], set[tuple[int, int, float]]]: + partial_bound_eval = _load_partial_bound_eval(bound_eval_path=bound_eval_path) + if not partial_bound_eval.empty: + partial_bound_eval = partial_bound_eval[ + partial_bound_eval["eval_random_state"].astype(int).isin(random_states) + ].reset_index(drop=True) + completed_keys = { + ( + int(row["candidate_rank"]), + int(row["eval_random_state"]), + float(row["alpha"]), ) - expected_checks = int(len(shortlist) * len(random_states) * len(alpha_grid)) - context["bound_total_checks"] = expected_checks - exact_threads = _context_exact_threads(context) - checkpoint_every = max(1, int(context.get("exact_checkpoint_every", 100))) + for row in partial_bound_eval.to_dict(orient="records") + } + bound_rows: list[dict[str, object]] = ( + partial_bound_eval.to_dict(orient="records") if not partial_bound_eval.empty else [] + ) + return bound_rows, completed_keys - bound_eval = _load_completed_bound_eval( - bound_eval_path=bound_eval_path, - expected_checks=expected_checks, + +def _load_aligned_datasets( + *, + context: dict[str, object], + plan: ExactEvalPlan, +) -> dict[int, pd.DataFrame]: + return { + int(seed): _load_aligned_dataset( + conformal_intervals_path=str(context["conformal_intervals_path"]), + max_candidates=int(plan.exact_max_candidates), + random_state=int(seed), + ) + for seed in plan.random_states + } + + +def _iter_pending_exact_tasks( + *, + shortlist: pd.DataFrame, + context: dict[str, object], + plan: ExactEvalPlan, + completed_keys: set[tuple[int, int, float]], +) -> Iterator[ExactEvalTask]: + for _, row in shortlist.iterrows(): + policy = _policy_from_row( + row, + solver_backend_override=str(context["exact_solver_backend"]), + ) + candidate_payload = row.to_dict() + candidate_rank = int(candidate_payload["candidate_rank"]) + for eval_seed in plan.random_states: + for alpha in plan.alpha_grid: + cache_key = (candidate_rank, int(eval_seed), float(alpha)) + if cache_key in completed_keys: + continue + yield ExactEvalTask( + cache_key=cache_key, + candidate_payload=candidate_payload, + policy=policy, + eval_random_state=int(eval_seed), + alpha=float(alpha), + ) + + +def _evaluate_exact_task( + *, + context: dict[str, object], + task: ExactEvalTask, + aligned_by_seed: dict[int, pd.DataFrame], + exact_threads: int, +) -> dict[str, object]: + result = _validate_single_alpha( + aligned_by_seed[int(task.eval_random_state)], + alpha=float(task.alpha), + policy=task.policy, + allocator_mode="exact", + budget=_as_float(context["budget"]), + t_eval=_as_float(context["t_eval"]), + threads=int(exact_threads), ) - if bound_eval is None: - partial_bound_eval = _load_partial_bound_eval(bound_eval_path=bound_eval_path) - if not partial_bound_eval.empty: - partial_bound_eval = partial_bound_eval[ - partial_bound_eval["eval_random_state"].astype(int).isin(random_states) - ].reset_index(drop=True) - completed_keys = { - (int(row.candidate_rank), int(row.eval_random_state), float(row.alpha)) - for row in partial_bound_eval.itertuples(index=False) - } - bound_rows: list[dict[str, object]] = ( - partial_bound_eval.to_dict(orient="records") if not partial_bound_eval.empty else [] + return { + "candidate_rank": _as_int(task.candidate_payload["candidate_rank"]), + "eval_random_state": int(task.eval_random_state), + "frontier_solver_backend": str(context["frontier_solver_backend"]), + "exact_solver_backend": str(context["exact_solver_backend"]), + **task.candidate_payload, + **result, + } + + +def _run_missing_exact_checks( + *, + context: dict[str, object], + paths: ExactEvalPaths, + shortlist: pd.DataFrame, + plan: ExactEvalPlan, + base_elapsed_sec: float, +) -> pd.DataFrame: + bound_rows, completed_keys = _resume_exact_rows( + bound_eval_path=paths.bound_eval_path, + random_states=plan.random_states, + ) + completed_checks = len(completed_keys) + context["resume_completed_checks"] = int(completed_checks) + if completed_checks: + _write_exact_status( + context=context, + base_elapsed_sec=base_elapsed_sec, + bound_completed_checks=completed_checks, + phase="exact_bound_running", + state="running", + extra=_plan_status_extra(plan, {"resume_cache_rows": int(completed_checks)}), ) - completed_checks = len(completed_keys) - context["resume_completed_checks"] = int(completed_checks) - if completed_checks: - _write_exact_status( + aligned_by_seed = _load_aligned_datasets(context=context, plan=plan) + + for task in _iter_pending_exact_tasks( + shortlist=shortlist, + context=context, + plan=plan, + completed_keys=completed_keys, + ): + bound_rows.append( + _evaluate_exact_task( context=context, - base_elapsed_sec=base_elapsed_sec, - bound_completed_checks=completed_checks, - phase="exact_bound_running", - state="running", - extra={ - "resume_cache_rows": int(completed_checks), - "checkpoint_every": int(checkpoint_every), - "exact_max_candidates": int(exact_max_candidates), - "exact_random_states": random_states, - "requested_exact_random_states": requested_random_states, - "full_universe_seed_deduped": bool(full_universe_seed_deduped), - "exact_threads": int(exact_threads), - }, + task=task, + aligned_by_seed=aligned_by_seed, + exact_threads=plan.exact_threads, ) - aligned_by_seed = { - int(seed): _load_aligned_dataset( - conformal_intervals_path=str(context["conformal_intervals_path"]), - max_candidates=int(exact_max_candidates), - random_state=int(seed), + ) + completed_checks += 1 + completed_keys.add(task.cache_key) + _write_exact_status( + context=context, + base_elapsed_sec=base_elapsed_sec, + bound_completed_checks=completed_checks, + phase="exact_bound_running", + state="running", + extra=_plan_status_extra( + plan, + { + "candidate_rank": _as_int(task.candidate_payload["candidate_rank"]), + "eval_random_state": int(task.eval_random_state), + "current_alpha": float(task.alpha), + }, + ), + ) + if completed_checks % plan.checkpoint_every == 0: + atomic_write_parquet( + pd.DataFrame(bound_rows), + paths.bound_eval_path, + index=False, ) - for seed in random_states - } + return pd.DataFrame(bound_rows) - for _, row in shortlist.iterrows(): - policy = _policy_from_row( - row, - solver_backend_override=str(context["exact_solver_backend"]), - ) - candidate_payload = row.to_dict() - for eval_seed in random_states: - aligned = aligned_by_seed[int(eval_seed)] - for alpha in alpha_grid: - cache_key = ( - int(candidate_payload["candidate_rank"]), - int(eval_seed), - float(alpha), - ) - if cache_key in completed_keys: - continue - result = _validate_single_alpha( - aligned, - alpha=float(alpha), - policy=policy, - allocator_mode="exact", - budget=float(context["budget"]), - t_eval=float(context["t_eval"]), - threads=int(exact_threads), - ) - bound_rows.append( - { - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "frontier_solver_backend": str(context["frontier_solver_backend"]), - "exact_solver_backend": str(context["exact_solver_backend"]), - **candidate_payload, - **result, - } - ) - completed_checks += 1 - completed_keys.add(cache_key) - _write_exact_status( - context=context, - base_elapsed_sec=base_elapsed_sec, - bound_completed_checks=completed_checks, - phase="exact_bound_running", - state="running", - extra={ - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "current_alpha": float(alpha), - "checkpoint_every": int(checkpoint_every), - "exact_max_candidates": int(exact_max_candidates), - "exact_random_states": random_states, - "requested_exact_random_states": requested_random_states, - "full_universe_seed_deduped": bool(full_universe_seed_deduped), - "exact_threads": int(exact_threads), - }, - ) - if completed_checks % checkpoint_every == 0: - atomic_write_parquet( - pd.DataFrame(bound_rows), - bound_eval_path, - index=False, - ) - - bound_eval = pd.DataFrame(bound_rows) - atomic_write_parquet(bound_eval, bound_eval_path, index=False) - shortlist_eval = _aggregate_exact_results(shortlist=shortlist, bound_eval=bound_eval) - region_payload = _region_summary(shortlist_eval, bound_eval) - selected = shortlist_eval.iloc[0].copy() - selected_policy = _policy_from_row( - selected, - solver_backend_override=str(context["exact_solver_backend"]), + +def _load_or_run_bound_eval( + *, + context: dict[str, object], + paths: ExactEvalPaths, + shortlist: pd.DataFrame, + plan: ExactEvalPlan, + base_elapsed_sec: float, +) -> pd.DataFrame: + bound_eval = _load_completed_bound_eval( + bound_eval_path=paths.bound_eval_path, + expected_checks=plan.expected_checks, ) + if bound_eval is not None: + return bound_eval + bound_eval = _run_missing_exact_checks( + context=context, + paths=paths, + shortlist=shortlist, + plan=plan, + base_elapsed_sec=base_elapsed_sec, + ) + atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) + return bound_eval - search_space_payload = dict(context["search_space"]) # type: ignore[arg-type] + +def _search_space_payload( + *, + context: dict[str, object], + alpha_grid: list[float], +) -> dict[str, object]: + search_space = context["search_space"] + if not isinstance(search_space, dict): + raise TypeError("context['search_space'] must be a mapping") + search_space_payload = dict(search_space) requested_search_alpha_grid = search_space_payload.get("alpha_grid") search_space_payload["alpha_grid"] = alpha_grid if requested_search_alpha_grid != alpha_grid: search_space_payload["requested_alpha_grid"] = requested_search_alpha_grid search_space_payload["effective_alpha_grid"] = alpha_grid + return search_space_payload - payload = { + +def _build_selection_payload( + *, + context: dict[str, object], + paths: ExactEvalPaths, + plan: ExactEvalPlan, + selected: pd.Series, + selected_policy: dict[str, object], + region_payload: dict[str, object], +) -> dict[str, object]: + return { "schema_version": SCHEMA_VERSION, "generated_at_utc": datetime.now(tz=UTC).isoformat(), "run_label": str(context["run_label"]), "conformal_intervals_path": _repo_relative(str(context["conformal_intervals_path"])), - "search_space": search_space_payload, + "search_space": _search_space_payload(context=context, alpha_grid=plan.alpha_grid), "selection_policy": context["selection_policy"], "selected_policy": selected_policy, "selected_metrics": selected.to_dict(), @@ -452,7 +635,7 @@ def main(argv: list[str] | None = None) -> int: "frontier_raw_path": _repo_relative(str(context["frontier_raw_path"])), "frontier_path": _repo_relative(str(context["frontier_path"])), "shortlist_path": _repo_relative(str(context["shortlist_path"])), - "shortlist_exact_path": _repo_relative(shortlist_exact_path), + "shortlist_exact_path": _repo_relative(paths.shortlist_exact_path), "bound_eval_path": _repo_relative(str(context["bound_eval_path"])), "region_summary_path": _repo_relative(str(context["region_summary_path"])), "robust_region_summary": region_payload, @@ -461,29 +644,49 @@ def main(argv: list[str] | None = None) -> int: "resource_snapshot_path": _repo_relative(str(context["resource_snapshot_path"])), "frontier_solver_backend": str(context["frontier_solver_backend"]), "exact_solver_backend": str(context["exact_solver_backend"]), - "exact_threads": int(exact_threads), - "requested_exact_random_states": requested_random_states, - "effective_exact_random_states": random_states, - "full_universe_seed_deduped": bool(full_universe_seed_deduped), + "exact_threads": int(plan.exact_threads), + "requested_exact_random_states": plan.requested_random_states, + "effective_exact_random_states": plan.random_states, + "full_universe_seed_deduped": bool(plan.full_universe_seed_deduped), } - atomic_write_parquet(shortlist_eval, shortlist_exact_path, index=False) + +def _write_selection_artifacts( + *, + context: dict[str, object], + paths: ExactEvalPaths, + shortlist_eval: pd.DataFrame, + region_payload: dict[str, object], + selection_payload: dict[str, object], +) -> None: + atomic_write_parquet(shortlist_eval, paths.shortlist_exact_path, index=False) atomic_write_json(Path(str(context["region_summary_path"])), region_payload) - atomic_write_json(selection_path, payload) + atomic_write_json(paths.selection_path, selection_payload) - resource_payload = json.loads(resource_snapshot_path.read_text(encoding="utf-8")) + +def _update_resource_snapshot(path: Path) -> None: + resource_payload = json.loads(path.read_text(encoding="utf-8")) resource_payload["exact_helper_python"] = _repo_relative(sys.executable) resource_payload["exact_helper_end"] = _resource_snapshot() - atomic_write_json(resource_snapshot_path, resource_payload) + atomic_write_json(path, resource_payload) + +def _write_final_status( + *, + context: dict[str, object], + paths: ExactEvalPaths, + base_elapsed_sec: float, + selection_payload: dict[str, object], + selected: pd.Series, +) -> None: final_payload = _write_exact_status( context=context, base_elapsed_sec=base_elapsed_sec, - bound_completed_checks=int(context["bound_total_checks"]), + bound_completed_checks=_as_int(context["bound_total_checks"]), phase="selection_complete", state="completed", extra={ - "selection_reason": str(payload["selection_reason"]), + "selection_reason": str(selection_payload["selection_reason"]), "selected_alpha01_exact_pass": bool(selected["alpha01_exact_pass"]), "selected_realized_total_return": float(selected["realized_total_return"]), }, @@ -492,7 +695,60 @@ def main(argv: list[str] | None = None) -> int: STAGE_NAME, "003_selection_complete", final_payload, - checkpoint_dir=checkpoint_dir, + checkpoint_dir=paths.checkpoint_dir, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + context = _load_context(Path(args.context_path).resolve()) + paths = _exact_eval_paths(context) + base_elapsed_sec = _initialize_exact_progress(context=context, status_path=paths.status_path) + + shortlist = pd.read_parquet(paths.shortlist_path) + plan = _exact_eval_plan(context=context, shortlist_rows=len(shortlist)) + _validate_alpha_grid_supported(plan.alpha_grid) + context["bound_total_checks"] = plan.expected_checks + + bound_eval = _load_or_run_bound_eval( + context=context, + paths=paths, + shortlist=shortlist, + plan=plan, + base_elapsed_sec=base_elapsed_sec, + ) + shortlist_eval = _aggregate_exact_results(shortlist=shortlist, bound_eval=bound_eval) + region_payload = _region_summary(shortlist_eval, bound_eval) + selected = shortlist_eval.iloc[0].copy() + selected_policy = _policy_from_row( + selected, + solver_backend_override=str(context["exact_solver_backend"]), + ) + + selection_payload = _build_selection_payload( + context=context, + paths=paths, + plan=plan, + selected=selected, + selected_policy=selected_policy, + region_payload=region_payload, + ) + _write_selection_artifacts( + context=context, + paths=paths, + shortlist_eval=shortlist_eval, + region_payload=region_payload, + selection_payload=selection_payload, + ) + _update_resource_snapshot(paths.resource_snapshot_path) + _write_final_status( + context=context, + paths=paths, + base_elapsed_sec=base_elapsed_sec, + selection_payload=selection_payload, + selected=selected, ) logger.info( diff --git a/scripts/search/run_portfolio_search.py b/scripts/search/run_portfolio_search.py index 5567f4b..7982599 100644 --- a/scripts/search/run_portfolio_search.py +++ b/scripts/search/run_portfolio_search.py @@ -1,20 +1,24 @@ -"""Organized search entrypoint for portfolio selector/tradeoff runs.""" +"""Retired portfolio search entrypoint. + +The former generic ``scripts.run_long_pipeline`` orchestrator was removed when +the IJDS paper lane was narrowed around frozen artifacts. Keep this file as a +readable stop sign for old commands instead of failing with an import error. +""" from __future__ import annotations import sys -from scripts.run_long_pipeline import main as _main - def main(argv: list[str] | None = None) -> int: - return _main( - argv, - default_pipeline_family="search_portfolio", - default_sampling_profile="champion64safe", - default_include_rapids=False, - default_include_notebooks=False, + _ = argv + sys.stderr.write( + "scripts/search/run_portfolio_search.py is retired. The submitted IJDS " + "claim uses the closed finite-grid frontier; use " + "scripts/search/run_pool93_ijds_local_refinement.py only for an explicitly " + "tagged isolated refinement, not as a default paper path.\n" ) + return 2 if __name__ == "__main__": diff --git a/scripts/search/run_regret_auditability_sandbox.py b/scripts/search/run_regret_auditability_sandbox.py index d7003a4..9e02ed7 100644 --- a/scripts/search/run_regret_auditability_sandbox.py +++ b/scripts/search/run_regret_auditability_sandbox.py @@ -24,7 +24,7 @@ from dataclasses import asdict, dataclass from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, cast import yaml @@ -32,6 +32,9 @@ sys.path.insert(0, str(ROOT)) from src.features.feature_config_io import load_feature_config # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + IJDS_DECLARED_ALPHA_GRID_CSV, +) from src.utils.pipeline_runtime import atomic_write_json # noqa: E402 @@ -83,6 +86,10 @@ def _env_str(name: str, default: str) -> str: "0.60,0.75,0.90,1.0", ) PORTFOLIO_RANDOM_STATES = _env_str("CRPTO_SANDBOX_PORTFOLIO_RANDOM_STATES", "42,52,62") +PORTFOLIO_ALPHA_GRID = _env_str( + "CRPTO_SANDBOX_PORTFOLIO_ALPHA_GRID", + IJDS_DECLARED_ALPHA_GRID_CSV, +) PORTFOLIO_MAX_CANDIDATES = _env_int("CRPTO_SANDBOX_PORTFOLIO_MAX_CANDIDATES", 100000) PORTFOLIO_SHORTLIST_TOP_K = _env_int("CRPTO_SANDBOX_PORTFOLIO_SHORTLIST_TOP_K", 1000) @@ -736,45 +743,37 @@ def _pd_warm_start_candidates( for index, row in enumerate(selected[:5], start=1): if not isinstance(row, Mapping): continue + best_params = row.get("best_params") _append_warm_start_candidate( rows, source=f"{previous_phase}:top_{index}:{row.get('lane_id', 'unknown')}", - params=row.get("best_params"), + params=cast(Mapping[str, Any], best_params) + if isinstance(best_params, Mapping) + else None, include_iterations=include_iterations, ) return rows -def write_pd_config_snapshot( +def _apply_pd_feature_profile_config( *, - artifact_root: Path, - run_tag: str, - feature_profile_name: str, - policy_name: str, - phase: str, - n_trials: int, - cpu_threads: int, - base_params_override: Mapping[str, Any] | None = None, -) -> Path: - """Write an external PD config snapshot for one feature/policy lane and phase.""" - config = _load_yaml(CHAMPION_PD_CONFIG_PATH) - feature_profile_path = _write_feature_profile_snapshot( - artifact_root=artifact_root, - profile_name=feature_profile_name, - ) - policy = _monotonic_policy_for_feature_profile( - policy_name=policy_name, - feature_profile_name=feature_profile_name, - feature_profile_path=feature_profile_path, - ) - lane = _lane_id(feature_profile_name, policy_name) - phase_root = artifact_root / "pd" / feature_profile_name / policy_name / phase - profile = FEATURE_PROFILES[feature_profile_name] - + config: dict[str, Any], + feature_profile_path: Path, + profile: Mapping[str, Any], +) -> None: config["feature_source"] = dict(config.get("feature_source", {}) or {}) config["feature_source"]["feature_config_path"] = str(feature_profile_path) config["stable_core"] = dict(config.get("stable_core", {}) or {}) config["stable_core"]["enabled"] = bool(profile.get("stable_core_enabled", False)) + + +def _apply_pd_model_params( + *, + config: dict[str, Any], + policy: Mapping[str, int], + cpu_threads: int, + base_params_override: Mapping[str, Any] | None, +) -> None: config["model"] = dict(config.get("model", {}) or {}) config["model"]["params"] = dict(config["model"].get("params", {}) or {}) if base_params_override: @@ -797,9 +796,24 @@ def write_pd_config_snapshot( "monotone_constraints": _format_monotone_constraints(policy), } ) + + +def _apply_venn_abers_calibration(config: dict[str, Any]) -> None: config["calibration"] = dict(config.get("calibration", {}) or {}) config["calibration"]["method"] = "venn_abers" config["calibration"]["candidates"] = ["venn_abers"] + + +def _apply_pd_hpo_config( + *, + config: dict[str, Any], + artifact_root: Path, + phase_root: Path, + run_tag: str, + phase: str, + lane: str, + n_trials: int, +) -> list[dict[str, Any]]: config["hpo"] = dict(config.get("hpo", {}) or {}) config["hpo"].update( { @@ -847,27 +861,34 @@ def write_pd_config_snapshot( "bootstrap_type": ["MVS", "Bernoulli"], "grow_policy": ["SymmetricTree"], } - config["validation"] = dict(config.get("validation", {}) or {}) + return warm_start + + +def _pd_validation_policy(phase: str) -> tuple[dict[str, Any], bool]: if phase == "pd-smoke": - seed_replay = {"enabled": True, "top_k_trials": 1, "seeds": [42]} - walk_forward_enabled = False - elif phase == "pd-broad": - seed_replay = {"enabled": True, "top_k_trials": 10, "seeds": [42, 52, 62]} - walk_forward_enabled = True - else: - seed_replay = { - "enabled": True, - "top_k_trials": 30, - "seeds": [42, 52, 62, 72, 82], - } - walk_forward_enabled = True + return {"enabled": True, "top_k_trials": 1, "seeds": [42]}, False + if phase == "pd-broad": + return {"enabled": True, "top_k_trials": 10, "seeds": [42, 52, 62]}, True + return { + "enabled": True, + "top_k_trials": 30, + "seeds": [42, 52, 62, 72, 82], + }, True + + +def _apply_pd_validation_config(*, config: dict[str, Any], phase: str) -> None: + seed_replay, walk_forward_enabled = _pd_validation_policy(phase) + config["validation"] = dict(config.get("validation", {}) or {}) config["validation"]["seed_replay"] = { **seed_replay, "prioritize_gate_pass": True, } config["validation"]["walk_forward"] = dict(config["validation"].get("walk_forward", {}) or {}) config["validation"]["walk_forward"]["enabled"] = walk_forward_enabled - config["output"] = { + + +def _pd_output_paths(phase_root: Path) -> dict[str, Any]: + return { "model_path": str(phase_root / "models" / "pd_model.cbm"), "default_model_path": str(phase_root / "models" / "pd_default.cbm"), "tuned_model_path": str(phase_root / "models" / "pd_tuned.cbm"), @@ -887,6 +908,9 @@ def write_pd_config_snapshot( "shap_dir": str(phase_root / "reports" / "shap"), "write_legacy_model_copy": False, } + + +def _apply_pd_decision_threshold_config(*, config: dict[str, Any], phase_root: Path) -> None: config["decision_threshold"] = dict(config.get("decision_threshold", {}) or {}) config["decision_threshold"]["enabled"] = False config["decision_threshold"]["fairness_policy_path"] = "" @@ -896,6 +920,20 @@ def write_pd_config_snapshot( config["decision_threshold"]["output_path_v2"] = str( phase_root / "models" / "decision_threshold_v2.json" ) + + +def _apply_pd_sandbox_metadata( + *, + config: dict[str, Any], + run_tag: str, + phase: str, + feature_profile_name: str, + policy_name: str, + lane: str, + policy: Mapping[str, int], + base_params_override: Mapping[str, Any] | None, + warm_start: list[dict[str, Any]], +) -> None: config["sandbox_search"] = { "run_tag": run_tag, "phase": phase, @@ -913,6 +951,69 @@ def write_pd_config_snapshot( "skip_diagnostic_exports": True, "skip_shap_export": True, } + + +def write_pd_config_snapshot( + *, + artifact_root: Path, + run_tag: str, + feature_profile_name: str, + policy_name: str, + phase: str, + n_trials: int, + cpu_threads: int, + base_params_override: Mapping[str, Any] | None = None, +) -> Path: + """Write an external PD config snapshot for one feature/policy lane and phase.""" + config = _load_yaml(CHAMPION_PD_CONFIG_PATH) + feature_profile_path = _write_feature_profile_snapshot( + artifact_root=artifact_root, + profile_name=feature_profile_name, + ) + policy = _monotonic_policy_for_feature_profile( + policy_name=policy_name, + feature_profile_name=feature_profile_name, + feature_profile_path=feature_profile_path, + ) + lane = _lane_id(feature_profile_name, policy_name) + phase_root = artifact_root / "pd" / feature_profile_name / policy_name / phase + profile = FEATURE_PROFILES[feature_profile_name] + + _apply_pd_feature_profile_config( + config=config, + feature_profile_path=feature_profile_path, + profile=profile, + ) + _apply_pd_model_params( + config=config, + policy=policy, + cpu_threads=cpu_threads, + base_params_override=base_params_override, + ) + _apply_venn_abers_calibration(config) + warm_start = _apply_pd_hpo_config( + config=config, + artifact_root=artifact_root, + phase_root=phase_root, + run_tag=run_tag, + phase=phase, + lane=lane, + n_trials=n_trials, + ) + _apply_pd_validation_config(config=config, phase=phase) + config["output"] = _pd_output_paths(phase_root) + _apply_pd_decision_threshold_config(config=config, phase_root=phase_root) + _apply_pd_sandbox_metadata( + config=config, + run_tag=run_tag, + phase=phase, + feature_profile_name=feature_profile_name, + policy_name=policy_name, + lane=lane, + policy=policy, + base_params_override=base_params_override, + warm_start=warm_start, + ) target = artifact_root / "configs" / f"pd_{lane}_{phase}.yaml" assert_safe_output_paths( value for value in config["output"].values() if isinstance(value, (str, Path)) @@ -1360,19 +1461,8 @@ def _resolve_pd_candidate_for_conformal(artifact_root: Path, artifact_name: str) return canonical[artifact_name] -def build_phase_commands( - *, - artifact_root: Path, - run_tag: str, - phase: str, - max_workers: int, - cpu_threads: int, -) -> list[PhaseCommand]: - """Build commands for the requested sandbox phase.""" - safe_tag = sanitize_tag(run_tag) - artifact_root = artifact_root.resolve() - commands: list[PhaseCommand] = [] - base_env = { +def _sandbox_base_env(artifact_root: Path, safe_tag: str) -> dict[str, str]: + return { "CRPTO_RUN_TAG": safe_tag, "PIPELINE_RUN_TAG": safe_tag, "RUN_TAG": safe_tag, @@ -1380,314 +1470,409 @@ def build_phase_commands( "GPU_REPLAY_ARTIFACT_ROOT": str(artifact_root), "CRPTO_SANDBOX_ARTIFACT_ROOT": str(artifact_root), } - selected_phases = ( - ["pd-smoke", "pd-broad", "pd-refine", "conformal", "portfolio", "metrics"] - if phase == "all" - else [phase] + + +def _phase_resources( + phase: str, + *, + max_workers: int, + cpu_threads: int, +) -> tuple[int, int]: + workers = int(max_workers) if int(max_workers) > 0 else _default_workers_for_phase(phase) + threads = int(cpu_threads) if int(cpu_threads) > 0 else _default_threads_for_phase(phase) + return workers, threads + + +def _pd_incumbent_command( + *, + artifact_root: Path, + safe_tag: str, + phase: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> PhaseCommand: + config_path = write_pd_incumbent_config_snapshot( + artifact_root=artifact_root, + run_tag=safe_tag, + phase=phase, + cpu_threads=phase_threads, ) - for selected_phase in selected_phases: - phase_workers = ( - int(max_workers) if int(max_workers) > 0 else _default_workers_for_phase(selected_phase) + output_root = artifact_root / "pd_baselines" / "champion" / phase + outputs = [ + output_root / "models" / "pd_model.cbm", + output_root / "models" / "pd_training_status.json", + output_root / "models" / "pd_training_record.pkl", + ] + command_name = f"{phase}_incumbent__frozen_champion" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(ROOT / "scripts" / "train_pd_model.py"), + "--config", + str(config_path), + "--hpo_enabled", + "false", + "--hpo_n_trials", + "0", + "--walk_forward_enabled", + "true" if phase != "pd-smoke" else "false", + "--seed_replay_enabled", + "false", + ], + outputs=[str(path) for path in outputs], + checkpoint=str(output_root / "models" / "pd_training_checkpoints"), + env=_command_env(base_env, phase_threads=phase_threads), + max_workers=phase_workers, + cpu_threads=phase_threads, + feature_profile="incumbent", + monotonic_policy="canonical_4", + lane_id="incumbent__frozen_champion", + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _pd_lane_command( + *, + artifact_root: Path, + safe_tag: str, + phase: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], + feature_profile_name: str, + policy_name: str, + n_trials: int, +) -> PhaseCommand: + lane = _lane_id(feature_profile_name, policy_name) + base_params = _previous_best_params_for_lane( + artifact_root=artifact_root, + phase=phase, + lane_id=lane, + ) + config_path = write_pd_config_snapshot( + artifact_root=artifact_root, + run_tag=safe_tag, + feature_profile_name=feature_profile_name, + policy_name=policy_name, + phase=phase, + n_trials=n_trials, + cpu_threads=phase_threads, + base_params_override=base_params, + ) + output_root = artifact_root / "pd" / feature_profile_name / policy_name / phase + remaining_trials = _pd_remaining_trials( + phase_root=output_root, + run_tag=safe_tag, + lane_id=lane, + phase=phase, + target_trials=n_trials, + ) + outputs = [ + output_root / "models" / "pd_model.cbm", + output_root / "models" / "pd_training_status.json", + output_root / "models" / "pd_hpo_seed_replay_status.json", + ] + command_name = f"{phase}_{lane}" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(ROOT / "scripts" / "train_pd_model.py"), + "--config", + str(config_path), + "--hpo_enabled", + "true", + "--hpo_n_trials", + str(remaining_trials), + "--walk_forward_enabled", + "true" if phase != "pd-smoke" else "false", + "--seed_replay_enabled", + "true", + ], + outputs=[str(path) for path in outputs], + checkpoint=str(output_root / "models" / "pd_training_checkpoints"), + env=_command_env(base_env, phase_threads=phase_threads), + max_workers=phase_workers, + cpu_threads=phase_threads, + feature_profile=feature_profile_name, + monotonic_policy=policy_name, + lane_id=lane, + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _build_pd_phase_commands( + *, + artifact_root: Path, + safe_tag: str, + phase: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> list[PhaseCommand]: + commands = [ + _pd_incumbent_command( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase=phase, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, ) - phase_threads = ( - int(cpu_threads) if int(cpu_threads) > 0 else _default_threads_for_phase(selected_phase) + ] + n_trials = _pd_trials_for_phase(phase) + commands.extend( + _pd_lane_command( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase=phase, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, + feature_profile_name=feature_profile_name, + policy_name=policy_name, + n_trials=n_trials, ) - if selected_phase in PD_PHASES: - incumbent_config_path = write_pd_incumbent_config_snapshot( + for feature_profile_name, policy_name in _selected_pd_lanes(artifact_root, phase) + ) + return commands + + +def _build_conformal_phase_command( + *, + artifact_root: Path, + safe_tag: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> PhaseCommand: + phase = "conformal" + conformal_root = artifact_root / phase / safe_tag + pd_model_path = _resolve_pd_candidate_for_conformal( + artifact_root, + "pd_shadow_canonical.cbm", + ) + pd_calibrator_path = _resolve_pd_candidate_for_conformal( + artifact_root, + "pd_shadow_calibrator.pkl", + ) + outputs = [ + conformal_root / "data" / "conformal_intervals_mondrian.parquet", + conformal_root / "models" / "conformal_results_mondrian.pkl", + conformal_root / "models" / "pd_conformal_width_attribution_status.json", + ] + command_name = "conformal_extensive_grid" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(ROOT / "scripts" / "generate_conformal_intervals.py"), + "--artifact_namespace", + safe_tag, + "--artifact_root", + str(artifact_root / phase), + "--model_override_path", + str(pd_model_path), + "--alpha_candidates_90", + "0.05,0.075,0.09,0.095,0.10,0.105,0.11,0.125,0.15,0.20", + "--alpha_candidates_95", + "0.025,0.04,0.045,0.05,0.055,0.06,0.075", + "--partition_candidates", + "grade,score_decile_mondrian,grade_x_scoreband_mondrian", + "--n_score_bins_candidates", + "5,10,15,20,30", + "--min_group_sizes", + "100,150,250,500,1000,2000", + "--score_scale_families", + "none,bernoulli_sqrt,bernoulli_sqrt_clipped_0.02,bernoulli_sqrt_clipped_0.05", + "--calibrator_override_path", + str(pd_calibrator_path), + ], + outputs=[str(path) for path in outputs], + checkpoint=str(conformal_root / "checkpoints"), + env=_command_env(base_env, phase_threads=phase_threads), + max_workers=phase_workers, + cpu_threads=phase_threads, + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _build_portfolio_phase_command( + *, + artifact_root: Path, + safe_tag: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> PhaseCommand: + phase = "portfolio" + portfolio_root = artifact_root / phase / safe_tag + conformal_path = ( + artifact_root / "conformal" / safe_tag / "data" / "conformal_intervals_mondrian.parquet" + ) + outputs = [ + portfolio_root / "data" / "portfolio_bound_aware_frontier.parquet", + portfolio_root / "data" / "portfolio_bound_aware_bound_eval.parquet", + portfolio_root / "models" / "portfolio_bound_aware_selection.json", + ] + command_name = "portfolio_extensive_frontier" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(ROOT / "scripts" / "search" / "run_portfolio_bound_aware_search.py"), + "--config", + str(ROOT / "configs" / "crpto_optimization.yaml"), + "--conformal-intervals-path", + str(conformal_path), + "--run-label", + safe_tag, + "--output-dir", + str(portfolio_root / "data"), + "--model-dir", + str(portfolio_root / "models"), + "--incumbent-policy-path", + str(CHAMPION_PORTFOLIO_POLICY_PATH), + "--incumbent-risk-neighbors", + "0.155,0.16,0.165,0.17,0.175,0.18", + "--incumbent-gamma-neighbors", + "0.425,0.45,0.475,0.50,0.525,0.55,0.575", + "--incumbent-policy-modes", + "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", + "--risk-grid", + PORTFOLIO_RISK_GRID, + "--gamma-grid", + PORTFOLIO_GAMMA_GRID, + "--aversion-grid", + PORTFOLIO_AVERSION_GRID, + "--delta-cap-grid", + PORTFOLIO_CAP_TAIL_GRID, + "--tail-focus-grid", + PORTFOLIO_CAP_TAIL_GRID, + "--policy-modes", + "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", + "--alpha-grid", + PORTFOLIO_ALPHA_GRID, + "--max-candidates", + str(PORTFOLIO_MAX_CANDIDATES), + "--shortlist-top-k", + str(PORTFOLIO_SHORTLIST_TOP_K), + "--random-states", + PORTFOLIO_RANDOM_STATES, + "--solver-backend", + "highs", + "--exact-solver-backend", + "highs", + ], + outputs=[str(path) for path in outputs], + checkpoint=str(portfolio_root / "models" / "portfolio_bound_aware_runtime_checkpoints"), + env=_command_env(base_env, phase_threads=phase_threads), + max_workers=phase_workers, + cpu_threads=phase_threads, + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _build_metrics_phase_command( + *, + artifact_root: Path, + safe_tag: str, + base_env: Mapping[str, str], +) -> PhaseCommand: + phase = "metrics" + metrics_path = artifact_root / phase / "frontier_metrics_manifest.json" + command_name = "metrics_manifest" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(Path(__file__).resolve()), + "--run-tag", + safe_tag, + "--artifact-root", + str(artifact_root), + "--phase", + "plan", + "--resume", + ], + outputs=[str(metrics_path)], + checkpoint=str(artifact_root / phase), + env=_command_env(base_env, phase_threads=1), + max_workers=1, + cpu_threads=1, + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _commands_for_phase( + *, + artifact_root: Path, + safe_tag: str, + phase: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> list[PhaseCommand]: + if phase in PD_PHASES: + return _build_pd_phase_commands( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase=phase, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, + ) + if phase == "conformal": + return [ + _build_conformal_phase_command( artifact_root=artifact_root, - run_tag=safe_tag, - phase=selected_phase, - cpu_threads=phase_threads, - ) - incumbent_root = artifact_root / "pd_baselines" / "champion" / selected_phase - incumbent_outputs = [ - incumbent_root / "models" / "pd_model.cbm", - incumbent_root / "models" / "pd_training_status.json", - incumbent_root / "models" / "pd_training_record.pkl", - ] - incumbent_command_name = f"{selected_phase}_incumbent__frozen_champion" - incumbent_stdout_log, incumbent_stderr_log = _command_log_files( - artifact_root, - selected_phase, - incumbent_command_name, - ) - commands.append( - PhaseCommand( - name=incumbent_command_name, - phase=selected_phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "train_pd_model.py"), - "--config", - str(incumbent_config_path), - "--hpo_enabled", - "false", - "--hpo_n_trials", - "0", - "--walk_forward_enabled", - "true" if selected_phase != "pd-smoke" else "false", - "--seed_replay_enabled", - "false", - ], - outputs=[str(path) for path in incumbent_outputs], - checkpoint=str(incumbent_root / "models" / "pd_training_checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - feature_profile="incumbent", - monotonic_policy="canonical_4", - lane_id="incumbent__frozen_champion", - stdout_log=str(incumbent_stdout_log), - stderr_log=str(incumbent_stderr_log), - ) - ) - n_trials = _pd_trials_for_phase(selected_phase) - for feature_profile_name, policy_name in _selected_pd_lanes( - artifact_root, - selected_phase, - ): - lane = _lane_id(feature_profile_name, policy_name) - base_params = _previous_best_params_for_lane( - artifact_root=artifact_root, - phase=selected_phase, - lane_id=lane, - ) - config_path = write_pd_config_snapshot( - artifact_root=artifact_root, - run_tag=safe_tag, - feature_profile_name=feature_profile_name, - policy_name=policy_name, - phase=selected_phase, - n_trials=n_trials, - cpu_threads=phase_threads, - base_params_override=base_params, - ) - output_root = ( - artifact_root / "pd" / feature_profile_name / policy_name / selected_phase - ) - remaining_trials = _pd_remaining_trials( - phase_root=output_root, - run_tag=safe_tag, - lane_id=lane, - phase=selected_phase, - target_trials=n_trials, - ) - outputs = [ - output_root / "models" / "pd_model.cbm", - output_root / "models" / "pd_training_status.json", - output_root / "models" / "pd_hpo_seed_replay_status.json", - ] - command_name = f"{selected_phase}_{lane}" - stdout_log, stderr_log = _command_log_files( - artifact_root, - selected_phase, - command_name, - ) - commands.append( - PhaseCommand( - name=command_name, - phase=selected_phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "train_pd_model.py"), - "--config", - str(config_path), - "--hpo_enabled", - "true", - "--hpo_n_trials", - str(remaining_trials), - "--walk_forward_enabled", - "true" if selected_phase != "pd-smoke" else "false", - "--seed_replay_enabled", - "true", - ], - outputs=[str(path) for path in outputs], - checkpoint=str(output_root / "models" / "pd_training_checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - feature_profile=feature_profile_name, - monotonic_policy=policy_name, - lane_id=lane, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - ) - elif selected_phase == "conformal": - conformal_root = artifact_root / "conformal" / safe_tag - pd_model_path = _resolve_pd_candidate_for_conformal( - artifact_root, - "pd_shadow_canonical.cbm", - ) - pd_calibrator_path = _resolve_pd_candidate_for_conformal( - artifact_root, - "pd_shadow_calibrator.pkl", - ) - outputs = [ - conformal_root / "data" / "conformal_intervals_mondrian.parquet", - conformal_root / "models" / "conformal_results_mondrian.pkl", - conformal_root / "models" / "pd_conformal_width_attribution_status.json", - ] - command_name = "conformal_extensive_grid" - stdout_log, stderr_log = _command_log_files( - artifact_root, - selected_phase, - command_name, - ) - commands.append( - PhaseCommand( - name=command_name, - phase=selected_phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "generate_conformal_intervals.py"), - "--artifact_namespace", - safe_tag, - "--artifact_root", - str(artifact_root / "conformal"), - "--model_override_path", - str(pd_model_path), - "--alpha_candidates_90", - "0.05,0.075,0.09,0.095,0.10,0.105,0.11,0.125,0.15,0.20", - "--alpha_candidates_95", - "0.025,0.04,0.045,0.05,0.055,0.06,0.075", - "--partition_candidates", - "grade,score_decile_mondrian,grade_x_scoreband_mondrian", - "--n_score_bins_candidates", - "5,10,15,20,30", - "--min_group_sizes", - "100,150,250,500,1000,2000", - "--score_scale_families", - "none,bernoulli_sqrt,bernoulli_sqrt_clipped_0.02,bernoulli_sqrt_clipped_0.05", - "--calibrator_override_path", - str(pd_calibrator_path), - ], - outputs=[str(path) for path in outputs], - checkpoint=str(conformal_root / "checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - ) - elif selected_phase == "portfolio": - portfolio_root = artifact_root / "portfolio" / safe_tag - conformal_path = ( - artifact_root - / "conformal" - / safe_tag - / "data" - / "conformal_intervals_mondrian.parquet" - ) - outputs = [ - portfolio_root / "data" / "portfolio_bound_aware_frontier.parquet", - portfolio_root / "data" / "portfolio_bound_aware_bound_eval.parquet", - portfolio_root / "models" / "portfolio_bound_aware_selection.json", - ] - command_name = "portfolio_extensive_frontier" - stdout_log, stderr_log = _command_log_files( - artifact_root, - selected_phase, - command_name, + safe_tag=safe_tag, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, ) - commands.append( - PhaseCommand( - name=command_name, - phase=selected_phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "search" / "run_portfolio_bound_aware_search.py"), - "--config", - str(ROOT / "configs" / "crpto_optimization.yaml"), - "--conformal-intervals-path", - str(conformal_path), - "--run-label", - safe_tag, - "--output-dir", - str(portfolio_root / "data"), - "--model-dir", - str(portfolio_root / "models"), - "--incumbent-policy-path", - str(CHAMPION_PORTFOLIO_POLICY_PATH), - "--incumbent-risk-neighbors", - "0.155,0.16,0.165,0.17,0.175,0.18", - "--incumbent-gamma-neighbors", - "0.425,0.45,0.475,0.50,0.525,0.55,0.575", - "--incumbent-policy-modes", - "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", - "--risk-grid", - PORTFOLIO_RISK_GRID, - "--gamma-grid", - PORTFOLIO_GAMMA_GRID, - "--aversion-grid", - PORTFOLIO_AVERSION_GRID, - "--delta-cap-grid", - PORTFOLIO_CAP_TAIL_GRID, - "--tail-focus-grid", - PORTFOLIO_CAP_TAIL_GRID, - "--policy-modes", - "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", - "--alpha-grid", - "0.01,0.02,0.03,0.05,0.10,0.15,0.20", - "--max-candidates", - str(PORTFOLIO_MAX_CANDIDATES), - "--shortlist-top-k", - str(PORTFOLIO_SHORTLIST_TOP_K), - "--random-states", - PORTFOLIO_RANDOM_STATES, - "--solver-backend", - "highs", - "--exact-solver-backend", - "highs", - ], - outputs=[str(path) for path in outputs], - checkpoint=str( - portfolio_root / "models" / "portfolio_bound_aware_runtime_checkpoints" - ), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - ) - elif selected_phase == "metrics": - metrics_path = artifact_root / "metrics" / "frontier_metrics_manifest.json" - command_name = "metrics_manifest" - stdout_log, stderr_log = _command_log_files( - artifact_root, - selected_phase, - command_name, + ] + if phase == "portfolio": + return [ + _build_portfolio_phase_command( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, ) - commands.append( - PhaseCommand( - name=command_name, - phase=selected_phase, - command=[ - sys.executable, - str(Path(__file__).resolve()), - "--run-tag", - safe_tag, - "--artifact-root", - str(artifact_root), - "--phase", - "plan", - "--resume", - ], - outputs=[str(metrics_path)], - checkpoint=str(artifact_root / "metrics"), - env=_command_env(base_env, phase_threads=1), - max_workers=1, - cpu_threads=1, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) + ] + if phase == "metrics": + return [ + _build_metrics_phase_command( + artifact_root=artifact_root, + safe_tag=safe_tag, + base_env=base_env, ) - elif selected_phase in {"plan", "deps"}: - continue - else: - raise ValueError(f"Unknown sandbox phase: {selected_phase}") + ] + if phase in {"plan", "deps"}: + return [] + raise ValueError(f"Unknown sandbox phase: {phase}") + + +def _validate_phase_command_paths(commands: Iterable[PhaseCommand]) -> None: for command in commands: assert_safe_output_paths(command.outputs) assert_safe_output_path(command.checkpoint) @@ -1695,6 +1880,43 @@ def build_phase_commands( assert_safe_output_path(command.stdout_log) if command.stderr_log: assert_safe_output_path(command.stderr_log) + + +def build_phase_commands( + *, + artifact_root: Path, + run_tag: str, + phase: str, + max_workers: int, + cpu_threads: int, +) -> list[PhaseCommand]: + """Build commands for the requested sandbox phase.""" + safe_tag = sanitize_tag(run_tag) + artifact_root = artifact_root.resolve() + base_env = _sandbox_base_env(artifact_root, safe_tag) + selected_phases = ( + ["pd-smoke", "pd-broad", "pd-refine", "conformal", "portfolio", "metrics"] + if phase == "all" + else [phase] + ) + commands: list[PhaseCommand] = [] + for selected_phase in selected_phases: + phase_workers, phase_threads = _phase_resources( + selected_phase, + max_workers=max_workers, + cpu_threads=cpu_threads, + ) + commands.extend( + _commands_for_phase( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase=selected_phase, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, + ) + ) + _validate_phase_command_paths(commands) return commands @@ -2052,6 +2274,160 @@ def _run_one_command(command: PhaseCommand) -> tuple[PhaseCommand, int]: return command, returncode +def _phase_command_groups(commands: Sequence[PhaseCommand]) -> list[list[PhaseCommand]]: + phase_groups: list[list[PhaseCommand]] = [] + for command in commands: + if not phase_groups or phase_groups[-1][0].phase != command.phase: + phase_groups.append([command]) + else: + phase_groups[-1].append(command) + return phase_groups + + +def _log_skipped_completed( + *, + artifact_root: Path, + log_path: Path, + command: PhaseCommand, +) -> None: + _append_command_log( + log_path, + { + "captured_at_utc": utc_now_iso(), + "phase": command.phase, + "name": command.name, + "state": "skipped_completed", + "returncode": 0, + "checkpoint": command.checkpoint, + "stdout_log": command.stdout_log, + "stderr_log": command.stderr_log, + }, + ) + _log_command_to_mlflow( + artifact_root=artifact_root, + command=command, + state="skipped_completed", + returncode=0, + ) + + +def _pending_commands_for_group( + *, + artifact_root: Path, + log_path: Path, + group: list[PhaseCommand], + resume: bool, +) -> tuple[deque[PhaseCommand], int]: + pending: deque[PhaseCommand] = deque() + skipped = 0 + for command in group: + if resume and _completed_outputs(command): + _log_skipped_completed( + artifact_root=artifact_root, + log_path=log_path, + command=command, + ) + skipped += 1 + continue + pending.append(command) + return pending, skipped + + +def _start_available_commands( + *, + artifact_root: Path, + log_path: Path, + pending: deque[PhaseCommand], + running: dict[Future[tuple[PhaseCommand, int]], PhaseCommand], + executor: ThreadPoolExecutor, + worker_limit: int, +) -> Path | None: + last_checkpoint: Path | None = None + while pending and len(running) < worker_limit and _resource_allows_launch(artifact_root): + command = pending.popleft() + last_checkpoint = Path(command.checkpoint) + future = executor.submit(_run_one_command, command) + running[future] = command + _append_command_log( + log_path, + { + "captured_at_utc": utc_now_iso(), + "phase": command.phase, + "name": command.name, + "state": "started", + "returncode": "", + "checkpoint": command.checkpoint, + "stdout_log": command.stdout_log, + "stderr_log": command.stderr_log, + }, + ) + return last_checkpoint + + +def _record_finished_command( + *, + artifact_root: Path, + log_path: Path, + finished_command: PhaseCommand, + returncode: int, + failed_commands: list[PhaseCommand], +) -> None: + state = "complete" if returncode == 0 else "failed" + _append_command_log( + log_path, + { + "captured_at_utc": utc_now_iso(), + "phase": finished_command.phase, + "name": finished_command.name, + "state": state, + "returncode": returncode, + "checkpoint": finished_command.checkpoint, + "stdout_log": finished_command.stdout_log, + "stderr_log": finished_command.stderr_log, + }, + ) + _log_command_to_mlflow( + artifact_root=artifact_root, + command=finished_command, + state=state, + returncode=returncode, + ) + if returncode != 0 and finished_command.phase in PD_PHASES: + failed_commands.append(finished_command) + elif returncode != 0: + raise RuntimeError( + f"Sandbox command failed ({finished_command.name}) with return code {returncode}" + ) + + +def _select_phase_winners_if_needed( + *, + artifact_root: Path, + group: list[PhaseCommand], + failed_commands: list[PhaseCommand], + completed: int, + total_commands: int, +) -> None: + if not group or group[0].phase not in PD_PHASES: + return + selection_path = _select_pd_phase_winners(artifact_root, group[0].phase) + if selection_path is None: + failed_names = ", ".join(command.name for command in failed_commands[:10]) + raise RuntimeError( + "No successful PD candidates available after phase " + f"{group[0].phase}. Failed lanes: {failed_names}" + ) + _write_heartbeat( + artifact_root=artifact_root, + phase=group[0].phase, + completed_units=completed, + total_units=total_commands, + current_best_metric=None, + last_checkpoint_path=selection_path, + state="selected_with_failures" if failed_commands else "selected", + ) + + def _run_commands( *, artifact_root: Path, @@ -2060,69 +2436,32 @@ def _run_commands( ) -> None: log_path = artifact_root / "command_log.csv" completed = 0 - - phase_groups: list[list[PhaseCommand]] = [] - for command in commands: - if not phase_groups or phase_groups[-1][0].phase != command.phase: - phase_groups.append([command]) - else: - phase_groups[-1].append(command) + phase_groups = _phase_command_groups(commands) for group in phase_groups: - pending: deque[PhaseCommand] = deque() - for command in group: - if resume and _completed_outputs(command): - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": command.phase, - "name": command.name, - "state": "skipped_completed", - "returncode": 0, - "checkpoint": command.checkpoint, - "stdout_log": command.stdout_log, - "stderr_log": command.stderr_log, - }, - ) - _log_command_to_mlflow( - artifact_root=artifact_root, - command=command, - state="skipped_completed", - returncode=0, - ) - completed += 1 - continue - pending.append(command) - + pending, skipped = _pending_commands_for_group( + artifact_root=artifact_root, + log_path=log_path, + group=group, + resume=resume, + ) + completed += skipped worker_limit = max(1, max((command.max_workers for command in group), default=1)) running: dict[Future[tuple[PhaseCommand, int]], PhaseCommand] = {} failed_commands: list[PhaseCommand] = [] last_checkpoint: Path | None = None with ThreadPoolExecutor(max_workers=worker_limit) as executor: while pending or running: - while ( - pending - and len(running) < worker_limit - and _resource_allows_launch(artifact_root) - ): - command = pending.popleft() - last_checkpoint = Path(command.checkpoint) - future = executor.submit(_run_one_command, command) - running[future] = command - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": command.phase, - "name": command.name, - "state": "started", - "returncode": "", - "checkpoint": command.checkpoint, - "stdout_log": command.stdout_log, - "stderr_log": command.stderr_log, - }, - ) + next_checkpoint = _start_available_commands( + artifact_root=artifact_root, + log_path=log_path, + pending=pending, + running=running, + executor=executor, + worker_limit=worker_limit, + ) + if next_checkpoint is not None: + last_checkpoint = next_checkpoint _write_heartbeat( artifact_root=artifact_root, phase=group[0].phase if group else "waiting_for_ram", @@ -2141,51 +2480,23 @@ def _run_commands( return_when=FIRST_COMPLETED, ) for future in done: - command = running.pop(future) + running.pop(future) finished_command, returncode = future.result() completed += 1 - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": finished_command.phase, - "name": finished_command.name, - "state": "complete" if returncode == 0 else "failed", - "returncode": returncode, - "checkpoint": finished_command.checkpoint, - "stdout_log": finished_command.stdout_log, - "stderr_log": finished_command.stderr_log, - }, - ) - _log_command_to_mlflow( + _record_finished_command( artifact_root=artifact_root, - command=finished_command, - state="complete" if returncode == 0 else "failed", + log_path=log_path, + finished_command=finished_command, returncode=returncode, + failed_commands=failed_commands, ) - if returncode != 0 and finished_command.phase in PD_PHASES: - failed_commands.append(finished_command) - elif returncode != 0: - raise RuntimeError( - f"Sandbox command failed ({command.name}) with return code {returncode}" - ) - if group and group[0].phase in PD_PHASES: - selection_path = _select_pd_phase_winners(artifact_root, group[0].phase) - if selection_path is None: - failed_names = ", ".join(command.name for command in failed_commands[:10]) - raise RuntimeError( - "No successful PD candidates available after phase " - f"{group[0].phase}. Failed lanes: {failed_names}" - ) - _write_heartbeat( - artifact_root=artifact_root, - phase=group[0].phase, - completed_units=completed, - total_units=len(commands), - current_best_metric=None, - last_checkpoint_path=selection_path, - state="selected_with_failures" if failed_commands else "selected", - ) + _select_phase_winners_if_needed( + artifact_root=artifact_root, + group=group, + failed_commands=failed_commands, + completed=completed, + total_commands=len(commands), + ) _write_heartbeat( artifact_root=artifact_root, phase=commands[-1].phase if commands else "plan", diff --git a/scripts/select_economic_portfolio_policy.py b/scripts/select_economic_portfolio_policy.py index 30939e6..f292d04 100644 --- a/scripts/select_economic_portfolio_policy.py +++ b/scripts/select_economic_portfolio_policy.py @@ -4,8 +4,9 @@ import argparse import json +from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, cast import numpy as np import pandas as pd @@ -21,12 +22,51 @@ _run_strategy, ) from src.evaluation.ab_testing import compare_strategies +from src.optimization.portfolio_model import solution_allocation_vector from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag from src.utils.script_helpers import artifact_path as _artifact_path, try_load_json SCHEMA_VERSION = "2026-03-10.1" +@dataclass(frozen=True) +class SelectionSettings: + top_k: int + selector_name: str + min_funded_ratio: float + min_total_allocated_ratio: float + min_breadth_score: float + breadth_weight_funded_ratio: float + breadth_weight_allocation_ratio: float + breadth_weight_allocation_similarity: float + max_por_pct: float + canonical_modes: set[str] + ab_like_top_m: int + ab_like_bootstrap_n: int + ab_like_seed: int + + +@dataclass(frozen=True) +class DecisionInputs: + common: dict[str, object] + default_flag: np.ndarray + loan_amnt: np.ndarray + int_rates: np.ndarray + pd_high: np.ndarray + total_budget: float + universe_source: str | None + scenario_meta: dict[str, Any] + + +@dataclass(frozen=True) +class SelectionResult: + selected: dict[str, Any] + selected_policy: dict[str, Any] + selector_outcome: str + fallback_applied: bool + fallback_reason: str | None + + def _policy_key(row: pd.Series) -> tuple[object, ...]: return ( str(row.get("policy_mode", "")), @@ -58,6 +98,77 @@ def _load_json(path: Path) -> dict[str, Any]: return try_load_json(path) +def _load_config(path: str) -> dict[str, Any]: + with open(path, encoding="utf-8") as f: + payload = yaml.safe_load(f) + return payload if isinstance(payload, dict) else {} + + +def _selection_settings(config: dict[str, Any]) -> SelectionSettings: + selection_cfg = dict(config.get("portfolio_selection", {}) or {}) + min_funded_ratio = float(selection_cfg.get("min_funded_ratio", 0.95)) + return SelectionSettings( + top_k=int(selection_cfg.get("actual_ab_top_k", 20)), + selector_name=str(selection_cfg.get("canonical_selector", "economic_actual_ab_v1")), + min_funded_ratio=min_funded_ratio, + min_total_allocated_ratio=float(selection_cfg.get("min_total_allocated_ratio", 0.98)), + min_breadth_score=float(selection_cfg.get("min_breadth_score", min_funded_ratio)), + breadth_weight_funded_ratio=float(selection_cfg.get("breadth_weight_funded_ratio", 0.5)), + breadth_weight_allocation_ratio=float( + selection_cfg.get("breadth_weight_allocation_ratio", 0.3) + ), + breadth_weight_allocation_similarity=float( + selection_cfg.get("breadth_weight_allocation_similarity", 0.2) + ), + max_por_pct=float(selection_cfg.get("max_price_of_robustness_pct", -15.0)), + canonical_modes={ + str(x) for x in selection_cfg.get("canonical_policy_modes", ["blended_uncertainty"]) + }, + ab_like_top_m=int(selection_cfg.get("ab_like_top_m", 8)), + ab_like_bootstrap_n=int(selection_cfg.get("ab_like_bootstrap_n", 200)), + ab_like_seed=int(selection_cfg.get("ab_like_seed", 42)), + ) + + +def _load_frontier(frontier_path: str) -> pd.DataFrame: + frontier = pd.read_parquet(_artifact_path(frontier_path)) + if frontier.empty: + raise ValueError("portfolio_robustness_frontier.parquet is empty") + return frontier + + +def _prepare_decision_inputs( + *, + config: dict[str, Any], + candidate_universe_path: str, + decision_scenario: str, +) -> DecisionInputs: + test_df = pd.read_parquet("data/processed/test_fe.parquet") + intervals = pd.read_parquet("data/processed/conformal_intervals_mondrian.parquet") + test_df, intervals, universe_source = _apply_candidate_universe( + test_df, + intervals, + candidate_universe_path=candidate_universe_path, + max_candidates=0, + ) + test_df, intervals, scenario_meta = _apply_decision_scenario( + test_df, + intervals, + decision_scenario=decision_scenario, + ) + common, default_flag, loan_amnt, int_rates, pd_high = _build_common_inputs(test_df, intervals) + return DecisionInputs( + common=common, + default_flag=default_flag, + loan_amnt=loan_amnt, + int_rates=int_rates, + pd_high=pd_high, + total_budget=float(config["portfolio"]["total_budget"]), + universe_source=universe_source, + scenario_meta=cast(dict[str, Any], scenario_meta), + ) + + def _dedupe_candidates(rows: list[pd.Series]) -> list[pd.Series]: out: list[pd.Series] = [] seen: set[tuple[object, ...]] = set() @@ -184,7 +295,7 @@ def _control_metrics_by_risk( default_flag=default_flag, lgd_val=0.45, ) - alloc = np.array([sol["allocation"][i] for i in range(len(loan_amnt))], dtype=float) + alloc = solution_allocation_vector(sol, len(loan_amnt)) controls[risk_tol] = { "solution": sol, "returns": returns, @@ -198,287 +309,287 @@ def _control_metrics_by_risk( return controls -def main( - config_path: str = "configs/optimization.yaml", - frontier_path: str = "data/processed/portfolio_robustness_frontier.parquet", - research_policy_path: str = "models/portfolio_research_policy.json", - champion_policy_path: str = "models/champion_portfolio_policy.json", - status_path: str = "models/champion_policy_selection_status.json", - candidate_universe_path: str = "data/processed/champion_candidate_universe.parquet", - run_tag: str | None = None, - solver_backend: str = "highs", - decision_scenario: str = "baseline", -) -> None: - with open(config_path, encoding="utf-8") as f: - config = yaml.safe_load(f) - selection_cfg = dict(config.get("portfolio_selection", {}) or {}) - top_k = int(selection_cfg.get("actual_ab_top_k", 20)) - selector_name = str(selection_cfg.get("canonical_selector", "economic_actual_ab_v1")) - min_funded_ratio = float(selection_cfg.get("min_funded_ratio", 0.95)) - min_total_allocated_ratio = float(selection_cfg.get("min_total_allocated_ratio", 0.98)) - min_breadth_score = float(selection_cfg.get("min_breadth_score", min_funded_ratio)) - breadth_weight_funded_ratio = float(selection_cfg.get("breadth_weight_funded_ratio", 0.5)) - breadth_weight_allocation_ratio = float( - selection_cfg.get("breadth_weight_allocation_ratio", 0.3) +def _evaluate_candidate_row( + *, + row: pd.Series, + inputs: DecisionInputs, + controls: dict[float, dict[str, Any]], + settings: SelectionSettings, + solver_backend: str, +) -> dict[str, Any]: + policy = _policy_from_row(row, source="economic_actual_ab_v1") + risk_tol = float(policy["risk_tolerance"]) + control = controls[risk_tol] + sol_b, _ = _run_strategy( + common=inputs.common, + robust=True, + robust_policy=policy, + total_budget=inputs.total_budget, + max_portfolio_pd=risk_tol, + solver_backend=solver_backend, ) - breadth_weight_allocation_similarity = float( - selection_cfg.get("breadth_weight_allocation_similarity", 0.2) + returns_b, metrics_b = _candidate_metrics( + solution=sol_b, + loan_amnt=inputs.loan_amnt, + int_rates=inputs.int_rates, + default_flag=inputs.default_flag, + lgd_val=0.45, ) - max_por_pct = float(selection_cfg.get("max_price_of_robustness_pct", -15.0)) - canonical_modes = { - str(x) for x in selection_cfg.get("canonical_policy_modes", ["blended_uncertainty"]) - } - ab_like_top_m = int(selection_cfg.get("ab_like_top_m", 8)) - ab_like_bootstrap_n = int(selection_cfg.get("ab_like_bootstrap_n", 200)) - ab_like_seed = int(selection_cfg.get("ab_like_seed", 42)) - - frontier = pd.read_parquet(_artifact_path(frontier_path)) - if frontier.empty: - raise ValueError("portfolio_robustness_frontier.parquet is empty") - - test_df = pd.read_parquet("data/processed/test_fe.parquet") - intervals = pd.read_parquet("data/processed/conformal_intervals_mondrian.parquet") - test_df, intervals, universe_source = _apply_candidate_universe( - test_df, - intervals, - candidate_universe_path=candidate_universe_path, - max_candidates=0, + control_metrics = control["metrics"] + returns_control = np.asarray(control["returns"], dtype=float) + diff_total_return = float(metrics_b["total_return"] - float(control_metrics["total_return"])) + return_delta_pct = float( + diff_total_return / (abs(float(control_metrics["total_return"])) + 1e-6) * 100.0 ) - test_df, intervals, scenario_meta = _apply_decision_scenario( - test_df, - intervals, - decision_scenario=decision_scenario, + tolerance_total_return = abs(float(control_metrics["total_return"])) * 0.05 + funded_ratio = float(metrics_b["n_funded"] / max(float(control_metrics["n_funded"]), 1.0)) + total_allocated_ratio = float( + metrics_b["total_allocated"] / max(float(control_metrics["total_allocated"]), 1.0) ) - common, default_flag, loan_amnt, int_rates, pd_high = _build_common_inputs(test_df, intervals) - total_budget = float(config["portfolio"]["total_budget"]) - controls = _control_metrics_by_risk( - common=common, - default_flag=default_flag, - loan_amnt=loan_amnt, - int_rates=int_rates, - risk_values=frontier["risk_tolerance"].tolist(), - total_budget=total_budget, - solver_backend=solver_backend, + alloc_b = solution_allocation_vector(sol_b, len(inputs.loan_amnt)) + allocation_similarity = _allocation_similarity(control["allocation"], alloc_b) + breadth_score = _breadth_score( + funded_ratio=funded_ratio, + total_allocated_ratio=total_allocated_ratio, + allocation_similarity=allocation_similarity, + weight_funded_ratio=settings.breadth_weight_funded_ratio, + weight_allocation_ratio=settings.breadth_weight_allocation_ratio, + weight_allocation_similarity=settings.breadth_weight_allocation_similarity, + ) + cand_worst_pd = float( + np.sum(alloc_b * inputs.loan_amnt * inputs.pd_high) + / (float(sol_b["total_allocated"]) + 1e-6) ) + return { + "policy": policy, + "risk_tolerance": risk_tol, + "passed_no_regression": bool(diff_total_return >= -tolerance_total_return), + "diff_total_return": diff_total_return, + "tolerance_total_return": tolerance_total_return, + "funded_ratio": funded_ratio, + "total_allocated_ratio": total_allocated_ratio, + "worst_case_pd_reduction_bps": float( + (float(control["worst_case_pd"]) - cand_worst_pd) * 1e4 + ), + "price_of_robustness_pct": float(min(return_delta_pct, 0.0)), + "return_delta_pct": return_delta_pct, + "frontier_price_of_robustness_pct": float(row.get("price_of_robustness_pct", 0.0)), + "return_per_funded_delta": float( + metrics_b["avg_return_per_funded"] - float(control_metrics["avg_return_per_funded"]) + ), + "allocation_similarity": allocation_similarity, + "breadth_score": breadth_score, + "n_funded_candidate": int(metrics_b["n_funded"]), + "n_funded_control": int(control_metrics["n_funded"]), + "total_return_candidate": float(metrics_b["total_return"]), + "total_return_control": float(control_metrics["total_return"]), + "eligible_hard_filters": False, + "_returns_candidate": returns_b, + "_returns_control": returns_control, + } - candidate_rows = _select_candidate_rows(frontier, top_k=top_k) - if not candidate_rows: - raise ValueError("No eligible canonical candidates found in frontier") - evaluated: list[dict[str, Any]] = [] - for row in candidate_rows: - policy = _policy_from_row(row, source="economic_actual_ab_v1") - risk_tol = float(policy["risk_tolerance"]) - control = controls[risk_tol] - sol_b, _ = _run_strategy( - common=common, - robust=True, - robust_policy=policy, - total_budget=total_budget, - max_portfolio_pd=risk_tol, +def _evaluate_candidate_rows( + *, + candidate_rows: list[pd.Series], + inputs: DecisionInputs, + controls: dict[float, dict[str, Any]], + settings: SelectionSettings, + solver_backend: str, +) -> list[dict[str, Any]]: + return [ + _evaluate_candidate_row( + row=row, + inputs=inputs, + controls=controls, + settings=settings, solver_backend=solver_backend, ) - returns_b, metrics_b = _candidate_metrics( - solution=sol_b, - loan_amnt=loan_amnt, - int_rates=int_rates, - default_flag=default_flag, - lgd_val=0.45, - ) - control_metrics = control["metrics"] - returns_control = np.asarray(control["returns"], dtype=float) - diff_total_return = float( - metrics_b["total_return"] - float(control_metrics["total_return"]) - ) - return_delta_pct = float( - diff_total_return / (abs(float(control_metrics["total_return"])) + 1e-6) * 100.0 - ) - actual_price_of_robustness_pct = float(min(return_delta_pct, 0.0)) - tolerance_total_return = abs(float(control_metrics["total_return"])) * 0.05 - passed_no_regression = bool(diff_total_return >= -tolerance_total_return) - funded_ratio = float(metrics_b["n_funded"] / max(float(control_metrics["n_funded"]), 1.0)) - total_allocated_ratio = float( - metrics_b["total_allocated"] / max(float(control_metrics["total_allocated"]), 1.0) - ) - alloc_b = np.array([sol_b["allocation"][i] for i in range(len(loan_amnt))], dtype=float) - allocation_similarity = _allocation_similarity(control["allocation"], alloc_b) - breadth_score = _breadth_score( - funded_ratio=funded_ratio, - total_allocated_ratio=total_allocated_ratio, - allocation_similarity=allocation_similarity, - weight_funded_ratio=breadth_weight_funded_ratio, - weight_allocation_ratio=breadth_weight_allocation_ratio, - weight_allocation_similarity=breadth_weight_allocation_similarity, - ) - cand_worst_pd = float( - np.sum(alloc_b * loan_amnt * pd_high) / (float(sol_b["total_allocated"]) + 1e-6) - ) - evaluated.append( - { - "policy": policy, - "risk_tolerance": risk_tol, - "passed_no_regression": passed_no_regression, - "diff_total_return": diff_total_return, - "tolerance_total_return": tolerance_total_return, - "funded_ratio": funded_ratio, - "total_allocated_ratio": total_allocated_ratio, - "worst_case_pd_reduction_bps": float( - (float(control["worst_case_pd"]) - cand_worst_pd) * 1e4 - ), - "price_of_robustness_pct": actual_price_of_robustness_pct, - "return_delta_pct": return_delta_pct, - "frontier_price_of_robustness_pct": float(row.get("price_of_robustness_pct", 0.0)), - "return_per_funded_delta": float( - metrics_b["avg_return_per_funded"] - - float(control_metrics["avg_return_per_funded"]) - ), - "allocation_similarity": allocation_similarity, - "breadth_score": breadth_score, - "n_funded_candidate": int(metrics_b["n_funded"]), - "n_funded_control": int(control_metrics["n_funded"]), - "total_return_candidate": float(metrics_b["total_return"]), - "total_return_control": float(control_metrics["total_return"]), - "eligible_hard_filters": False, - "_returns_candidate": returns_b, - "_returns_control": returns_control, - } - ) + for row in candidate_rows + ] + + +def _base_hard_filters(item: dict[str, Any], settings: SelectionSettings) -> bool: + return bool( + item["passed_no_regression"] + and float(item["price_of_robustness_pct"]) >= settings.max_por_pct + and str(item["policy"]["policy_mode"]) in settings.canonical_modes + ) + +def _mark_hard_filter_eligibility( + evaluated: list[dict[str, Any]], + settings: SelectionSettings, +) -> None: for item in evaluated: - base_filters = bool( - item["passed_no_regression"] - and float(item["price_of_robustness_pct"]) >= max_por_pct - and str(item["policy"]["policy_mode"]) in canonical_modes - ) - if selector_name in {"economic_actual_ab_v2", "economic_actual_ab_v3"}: + base_filters = _base_hard_filters(item, settings) + if settings.selector_name in {"economic_actual_ab_v2", "economic_actual_ab_v3"}: item["eligible_hard_filters"] = bool( base_filters - and float(item["total_allocated_ratio"]) >= min_total_allocated_ratio - and float(item["breadth_score"]) >= min_breadth_score - and float(item["funded_ratio"]) >= min_funded_ratio + and float(item["total_allocated_ratio"]) >= settings.min_total_allocated_ratio + and float(item["breadth_score"]) >= settings.min_breadth_score + and float(item["funded_ratio"]) >= settings.min_funded_ratio ) else: item["eligible_hard_filters"] = bool( - base_filters and float(item["funded_ratio"]) >= min_funded_ratio + base_filters and float(item["funded_ratio"]) >= settings.min_funded_ratio ) + +def _base_rank_key(item: dict[str, Any]) -> tuple[float, float, float, float, float]: + return ( + float(item["worst_case_pd_reduction_bps"]), + float(item["diff_total_return"]), + float(item.get("breadth_score", 0.0)), + -abs(float(item["price_of_robustness_pct"])), + float(item["funded_ratio"]), + ) + + +def _v3_rank_key(item: dict[str, Any]) -> tuple[bool, float, float, float, float, float]: + return ( + bool(item.get("ab_like_passed_no_regression", False)), + float(item.get("ab_like_diff_total_return", item["diff_total_return"])), + float(item["worst_case_pd_reduction_bps"]), + float(item.get("breadth_score", 0.0)), + -abs(float(item["price_of_robustness_pct"])), + float(item["funded_ratio"]), + ) + + +def _candidate_pool_after_filters(evaluated: list[dict[str, Any]]) -> list[dict[str, Any]]: eligible = [x for x in evaluated if bool(x["eligible_hard_filters"])] robust_eligible = [x for x in eligible if float(x["policy"]["gamma"]) > 0.0] - candidate_pool = robust_eligible or eligible - - if selector_name == "economic_actual_ab_v3" and candidate_pool: - pre_ranked = sorted( - candidate_pool, - key=lambda x: ( - float(x["worst_case_pd_reduction_bps"]), - float(x["diff_total_return"]), - float(x.get("breadth_score", 0.0)), - -abs(float(x["price_of_robustness_pct"])), - float(x["funded_ratio"]), - ), - reverse=True, - )[: max(1, ab_like_top_m)] - for idx, item in enumerate(pre_ranked): - item.update( - _ab_like_score( - returns_control=np.asarray(item["_returns_control"], dtype=float), - returns_candidate=np.asarray(item["_returns_candidate"], dtype=float), - seed=ab_like_seed + idx, - n_boot=ab_like_bootstrap_n, - ) - ) - robust_ab_like = [ - x for x in pre_ranked if bool(x.get("ab_like_passed_no_regression", False)) - ] - candidate_pool = robust_ab_like or pre_ranked - - fallback_applied = False - fallback_reason = None - selector_outcome = "robust_selected" - if candidate_pool: - if selector_name == "economic_actual_ab_v3": - selected = sorted( - candidate_pool, - key=lambda x: ( - bool(x.get("ab_like_passed_no_regression", False)), - float(x.get("ab_like_diff_total_return", x["diff_total_return"])), - float(x["worst_case_pd_reduction_bps"]), - float(x.get("breadth_score", 0.0)), - -abs(float(x["price_of_robustness_pct"])), - float(x["funded_ratio"]), - ), - reverse=True, - )[0] - else: - selected = sorted( - candidate_pool, - key=lambda x: ( - float(x["worst_case_pd_reduction_bps"]), - float(x["diff_total_return"]), - float(x.get("breadth_score", 0.0)), - -abs(float(x["price_of_robustness_pct"])), - float(x["funded_ratio"]), - ), - reverse=True, - )[0] - if float(selected["policy"]["gamma"]) <= 0.0: - selector_outcome = "fallback_nonrobust" - fallback_applied = True - fallback_reason = "no_economically_viable_robust_policy" - else: - fallback_applied = True - selector_outcome = "fallback_nonrobust" - fallback_reason = "no_economically_viable_robust_policy" - fallback_row = frontier.loc[frontier["selected_for_champion"].fillna(False).astype(bool)] - selected_row = fallback_row.iloc[0] if not fallback_row.empty else frontier.iloc[0] - selected = { - "policy": { - **_policy_from_row(selected_row, source="economic_actual_ab_v1_fallback"), - "gamma": 0.0, - "policy_mode": "blended_uncertainty", - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - }, - "risk_tolerance": float(selected_row["risk_tolerance"]), - "passed_no_regression": True, - "diff_total_return": 0.0, - "tolerance_total_return": abs( - float(controls[float(selected_row["risk_tolerance"])]["metrics"]["total_return"]) + return robust_eligible or eligible + + +def _apply_ab_like_screen( + candidate_pool: list[dict[str, Any]], + settings: SelectionSettings, +) -> list[dict[str, Any]]: + if settings.selector_name != "economic_actual_ab_v3" or not candidate_pool: + return candidate_pool + pre_ranked = sorted(candidate_pool, key=_base_rank_key, reverse=True)[ + : max(1, settings.ab_like_top_m) + ] + for idx, item in enumerate(pre_ranked): + item.update( + _ab_like_score( + returns_control=np.asarray(item["_returns_control"], dtype=float), + returns_candidate=np.asarray(item["_returns_candidate"], dtype=float), + seed=settings.ab_like_seed + idx, + n_boot=settings.ab_like_bootstrap_n, ) - * 0.05, - "funded_ratio": 1.0, - "total_allocated_ratio": 1.0, - "worst_case_pd_reduction_bps": 0.0, - "price_of_robustness_pct": 0.0, - "return_per_funded_delta": 0.0, - "allocation_similarity": 1.0, - "breadth_score": 1.0, - "n_funded_candidate": int( - controls[float(selected_row["risk_tolerance"])]["metrics"]["n_funded"] - ), - "n_funded_control": int( - controls[float(selected_row["risk_tolerance"])]["metrics"]["n_funded"] - ), - "total_return_candidate": float( - controls[float(selected_row["risk_tolerance"])]["metrics"]["total_return"] - ), - "total_return_control": float( - controls[float(selected_row["risk_tolerance"])]["metrics"]["total_return"] - ), - "eligible_hard_filters": False, - } + ) + robust_ab_like = [x for x in pre_ranked if bool(x.get("ab_like_passed_no_regression", False))] + return robust_ab_like or pre_ranked + + +def _select_ranked_candidate( + candidate_pool: list[dict[str, Any]], + settings: SelectionSettings, +) -> dict[str, Any]: + rank_key = _v3_rank_key if settings.selector_name == "economic_actual_ab_v3" else _base_rank_key + return sorted(candidate_pool, key=rank_key, reverse=True)[0] + + +def _fallback_selected_candidate( + frontier: pd.DataFrame, + controls: dict[float, dict[str, Any]], +) -> dict[str, Any]: + fallback_row = frontier.loc[frontier["selected_for_champion"].fillna(False).astype(bool)] + selected_row = fallback_row.iloc[0] if not fallback_row.empty else frontier.iloc[0] + risk_tol = float(selected_row["risk_tolerance"]) + control_metrics = controls[risk_tol]["metrics"] + return { + "policy": { + **_policy_from_row(selected_row, source="economic_actual_ab_v1_fallback"), + "gamma": 0.0, + "policy_mode": "blended_uncertainty", + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + }, + "risk_tolerance": risk_tol, + "passed_no_regression": True, + "diff_total_return": 0.0, + "tolerance_total_return": abs(float(control_metrics["total_return"])) * 0.05, + "funded_ratio": 1.0, + "total_allocated_ratio": 1.0, + "worst_case_pd_reduction_bps": 0.0, + "price_of_robustness_pct": 0.0, + "return_per_funded_delta": 0.0, + "allocation_similarity": 1.0, + "breadth_score": 1.0, + "n_funded_candidate": int(control_metrics["n_funded"]), + "n_funded_control": int(control_metrics["n_funded"]), + "total_return_candidate": float(control_metrics["total_return"]), + "total_return_control": float(control_metrics["total_return"]), + "eligible_hard_filters": False, + } - resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - research_policy = _load_json(_artifact_path(research_policy_path)) - champion_payload = { - "selection_stage": selector_name, - "selection_universe_path": universe_source or str(_artifact_path(candidate_universe_path)), + +def _choose_selected_candidate( + *, + frontier: pd.DataFrame, + controls: dict[float, dict[str, Any]], + evaluated: list[dict[str, Any]], + settings: SelectionSettings, +) -> SelectionResult: + _mark_hard_filter_eligibility(evaluated, settings) + candidate_pool = _apply_ab_like_screen(_candidate_pool_after_filters(evaluated), settings) + fallback_reason = "no_economically_viable_robust_policy" + if not candidate_pool: + selected = _fallback_selected_candidate(frontier, controls) + return SelectionResult( + selected=selected, + selected_policy=cast(dict[str, Any], selected["policy"]), + selector_outcome="fallback_nonrobust", + fallback_applied=True, + fallback_reason=fallback_reason, + ) + + selected = _select_ranked_candidate(candidate_pool, settings) + selected_policy = cast(dict[str, Any], selected["policy"]) + if float(selected_policy["gamma"]) <= 0.0: + return SelectionResult( + selected=selected, + selected_policy=selected_policy, + selector_outcome="fallback_nonrobust", + fallback_applied=True, + fallback_reason=fallback_reason, + ) + return SelectionResult( + selected=selected, + selected_policy=selected_policy, + selector_outcome="robust_selected", + fallback_applied=False, + fallback_reason=None, + ) + + +def _without_internal_returns(item: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in item.items() if not str(k).startswith("_returns_")} + + +def _build_champion_payload( + *, + settings: SelectionSettings, + selection: SelectionResult, + research_policy: dict[str, Any], + universe_path: str, + decision_scenario: str, + resolved_run_tag: str, +) -> dict[str, Any]: + selected = selection.selected + return { + "selection_stage": settings.selector_name, + "selection_universe_path": universe_path, "decision_scenario": str(decision_scenario), - "selection_outcome": selector_outcome, - "selected_policy": selected["policy"], + "selection_outcome": selection.selector_outcome, + "selected_policy": selection.selected_policy, "economic_metrics": { "diff_total_return": float(selected["diff_total_return"]), "passed_no_regression": bool(selected["passed_no_regression"]), @@ -504,21 +615,30 @@ def main( require_explicit=True, ), } - selected_clean = {k: v for k, v in selected.items() if not str(k).startswith("_returns_")} - status_payload = { - "selector_name": selector_name, - "universe_path": universe_source or str(_artifact_path(candidate_universe_path)), + + +def _build_status_payload( + *, + settings: SelectionSettings, + selection: SelectionResult, + inputs: DecisionInputs, + evaluated: list[dict[str, Any]], + controls: dict[float, dict[str, Any]], + universe_path: str, + decision_scenario: str, + resolved_run_tag: str, +) -> dict[str, Any]: + return { + "selector_name": settings.selector_name, + "universe_path": universe_path, "decision_scenario": str(decision_scenario), - "decision_scenario_meta": scenario_meta, + "decision_scenario_meta": inputs.scenario_meta, "control_metrics": {str(k): dict(v["metrics"]) for k, v in controls.items()}, - "evaluated_candidates": [ - {k: v for k, v in item.items() if not str(k).startswith("_returns_")} - for item in evaluated - ], - "selected_candidate": selected_clean, - "selector_outcome": selector_outcome, - "fallback_applied": fallback_applied, - "fallback_reason": fallback_reason, + "evaluated_candidates": [_without_internal_returns(item) for item in evaluated], + "selected_candidate": _without_internal_returns(selection.selected), + "selector_outcome": selection.selector_outcome, + "fallback_applied": selection.fallback_applied, + "fallback_reason": selection.fallback_reason, **build_artifact_metadata( schema_version=SCHEMA_VERSION, run_tag=resolved_run_tag, @@ -526,6 +646,14 @@ def main( ), } + +def _write_selection_outputs( + *, + champion_policy_path: str, + status_path: str, + champion_payload: dict[str, Any], + status_payload: dict[str, Any], +) -> None: champion_out = _artifact_path(champion_policy_path) status_out = _artifact_path(status_path) champion_out.parent.mkdir(parents=True, exist_ok=True) @@ -536,6 +664,81 @@ def main( logger.info("Saved champion policy selection status: {}", status_out) +def main( + config_path: str = "configs/optimization.yaml", + frontier_path: str = "data/processed/portfolio_robustness_frontier.parquet", + research_policy_path: str = "models/portfolio_research_policy.json", + champion_policy_path: str = "models/champion_portfolio_policy.json", + status_path: str = "models/champion_policy_selection_status.json", + candidate_universe_path: str = "data/processed/champion_candidate_universe.parquet", + run_tag: str | None = None, + solver_backend: str = "highs", + decision_scenario: str = "baseline", +) -> None: + config = _load_config(config_path) + settings = _selection_settings(config) + frontier = _load_frontier(frontier_path) + inputs = _prepare_decision_inputs( + config=config, + candidate_universe_path=candidate_universe_path, + decision_scenario=decision_scenario, + ) + controls = _control_metrics_by_risk( + common=inputs.common, + default_flag=inputs.default_flag, + loan_amnt=inputs.loan_amnt, + int_rates=inputs.int_rates, + risk_values=frontier["risk_tolerance"].tolist(), + total_budget=inputs.total_budget, + solver_backend=solver_backend, + ) + + candidate_rows = _select_candidate_rows(frontier, top_k=settings.top_k) + if not candidate_rows: + raise ValueError("No eligible canonical candidates found in frontier") + + evaluated = _evaluate_candidate_rows( + candidate_rows=candidate_rows, + inputs=inputs, + controls=controls, + settings=settings, + solver_backend=solver_backend, + ) + selection = _choose_selected_candidate( + frontier=frontier, + controls=controls, + evaluated=evaluated, + settings=settings, + ) + resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) + research_policy = _load_json(_artifact_path(research_policy_path)) + universe_path = inputs.universe_source or str(_artifact_path(candidate_universe_path)) + champion_payload = _build_champion_payload( + settings=settings, + selection=selection, + research_policy=research_policy, + universe_path=universe_path, + decision_scenario=decision_scenario, + resolved_run_tag=resolved_run_tag, + ) + status_payload = _build_status_payload( + settings=settings, + selection=selection, + inputs=inputs, + evaluated=evaluated, + controls=controls, + universe_path=universe_path, + decision_scenario=decision_scenario, + resolved_run_tag=resolved_run_tag, + ) + _write_selection_outputs( + champion_policy_path=champion_policy_path, + status_path=status_path, + champion_payload=champion_payload, + status_payload=status_payload, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--config", default="configs/optimization.yaml") diff --git a/scripts/simulate_ab_test.py b/scripts/simulate_ab_test.py index 8c2d681..7ed3a8f 100644 --- a/scripts/simulate_ab_test.py +++ b/scripts/simulate_ab_test.py @@ -30,16 +30,203 @@ from loguru import logger from src.evaluation.ab_testing import ab_summary, compare_strategies -from src.optimization.portfolio_model import ( - compute_effective_pd, - optimize_portfolio_allocation, -) +from src.optimization.policy_evaluation import solve_policy_allocation from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag from src.utils.script_helpers import artifact_path as _artifact_path SCHEMA_VERSION = "2026-03-01.1" +def _default_robust_policy(max_portfolio_pd: float) -> dict[str, Any]: + return { + "source": "fallback_default", + "risk_tolerance": float(max_portfolio_pd), + "uncertainty_aversion": 0.0, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "hard_worst_case", + "gamma": 1.0, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + } + + +def _select_champion_policy(payload: dict[str, Any], policy_selector: str) -> dict[str, Any]: + if policy_selector == "robustness_aware": + return cast( + dict[str, Any], + payload.get("selected_policy_robustness_aware") or payload.get("selected_policy", {}), + ) + if policy_selector == "balanced_robustness": + return cast( + dict[str, Any], + payload.get("selected_policy_balanced_robustness") + or payload.get("selected_policy_guardrail_robustness") + or payload.get("selected_policy_robustness_aware") + or payload.get("selected_policy", {}), + ) + if policy_selector == "guardrail_robustness": + return cast( + dict[str, Any], + payload.get("selected_policy_guardrail_robustness") + or payload.get("selected_policy_balanced_robustness") + or payload.get("selected_policy_robustness_aware") + or payload.get("selected_policy", {}), + ) + if policy_selector == "explicit_champion_only": + selected = payload.get("selected_policy", {}) + if not selected: + raise ValueError("Champion policy artifact missing selected_policy") + return cast(dict[str, Any], selected) + return cast(dict[str, Any], payload.get("selected_policy", {})) + + +def _policy_from_selected_champion( + selected: dict[str, Any], + *, + max_portfolio_pd: float, + policy_selector: str, +) -> dict[str, Any]: + return { + "source": f"champion_policy_artifact::{policy_selector}", + "risk_tolerance": float(selected.get("risk_tolerance", max_portfolio_pd)), + "uncertainty_aversion": float(selected.get("uncertainty_aversion", 0.0)), + "min_budget_utilization": float(selected.get("min_budget_utilization", 0.0)), + "pd_cap_slack_penalty": float(selected.get("pd_cap_slack_penalty", 0.0)), + "policy_mode": str(selected.get("policy_mode", "hard_worst_case")), + "gamma": float(selected.get("gamma", 1.0)), + "delta_cap_quantile": float(selected.get("delta_cap_quantile", 1.0)), + "tail_focus_quantile": float(selected.get("tail_focus_quantile", 1.0)), + } + + +def _resolve_champion_robust_policy( + *, + champion_path: Path, + max_portfolio_pd: float, + policy_selector: str, +) -> dict[str, Any] | None: + if not champion_path.exists(): + if policy_selector == "explicit_champion_only": + raise FileNotFoundError(f"Missing champion portfolio policy artifact: {champion_path}") + return None + try: + payload_raw = json.loads(champion_path.read_text(encoding="utf-8")) + selected = ( + _select_champion_policy(payload_raw, policy_selector) + if isinstance(payload_raw, dict) + else {} + ) + policy = _policy_from_selected_champion( + selected, + max_portfolio_pd=max_portfolio_pd, + policy_selector=policy_selector, + ) + except Exception as exc: + if policy_selector == "explicit_champion_only": + raise + logger.warning( + f"Could not parse champion portfolio policy ({champion_path}): {exc}. " + "Falling back to summary-based policy." + ) + return None + logger.info( + "Resolved robust policy from champion artifact: " + f"risk_tolerance={policy['risk_tolerance']:.4f}, " + f"policy_mode={policy['policy_mode']}, gamma={policy['gamma']:.2f}" + ) + return policy + + +def _load_robustness_summary(path: Path) -> pd.DataFrame | None: + if not path.exists(): + logger.warning(f"Robustness summary not found ({path}); using fallback robust policy.") + return None + try: + return pd.read_parquet(path) + except Exception as exc: + logger.warning(f"Could not read robustness summary ({path}): {exc}") + return None + + +def _valid_summary_rows(summary: pd.DataFrame, required_cols: set[str]) -> pd.DataFrame | None: + if summary.empty or not required_cols.issubset(set(summary.columns)): + missing = sorted(required_cols - set(summary.columns)) + logger.warning( + "Robustness summary missing required columns or empty; " + f"missing={missing}. Using fallback robust policy." + ) + return None + work = summary.copy() + for col in required_cols: + work[col] = pd.to_numeric(work[col], errors="coerce") + work = work.dropna(subset=list(required_cols)).reset_index(drop=True) + if work.empty: + logger.warning("No valid numeric robust summary rows; using fallback policy.") + return None + return work + + +def _best_summary_policy_row(work: pd.DataFrame, target: float) -> pd.Series: + lower_eq = work.loc[work["risk_tolerance"] <= target + 1e-12].copy() + candidate_pool = lower_eq if not lower_eq.empty else work + candidate_pool["_distance"] = (candidate_pool["risk_tolerance"] - target).abs() + if "best_robust_return" not in candidate_pool.columns: + return candidate_pool.sort_values(by=["_distance"], ascending=[True]).iloc[0] + candidate_pool["best_robust_return"] = pd.to_numeric( + candidate_pool["best_robust_return"], errors="coerce" + ).fillna(float("-inf")) + return candidate_pool.sort_values( + by=["_distance", "best_robust_return"], + ascending=[True, False], + ).iloc[0] + + +def _policy_from_summary_row(row: pd.Series) -> dict[str, Any]: + return { + "source": "portfolio_robustness_summary", + "risk_tolerance": float(row["risk_tolerance"]), + "uncertainty_aversion": float(row["best_robust_lambda"]), + "min_budget_utilization": float(row["best_robust_min_budget_utilization"]), + "pd_cap_slack_penalty": float(row["best_robust_pd_cap_slack_penalty"]), + "policy_mode": str(row.get("best_robust_policy_mode", "hard_worst_case")), + "gamma": float(row.get("best_robust_gamma", 1.0)), + "delta_cap_quantile": float(row.get("best_robust_delta_cap_quantile", 1.0)), + } + + +def _resolve_summary_robust_policy( + *, + path: Path, + max_portfolio_pd: float, +) -> dict[str, Any] | None: + summary = _load_robustness_summary(path) + if summary is None: + return None + + required_cols = { + "risk_tolerance", + "best_robust_lambda", + "best_robust_min_budget_utilization", + "best_robust_pd_cap_slack_penalty", + } + work = _valid_summary_rows(summary, required_cols) + if work is None: + return None + + policy = _policy_from_summary_row( + _best_summary_policy_row(work, target=float(max_portfolio_pd)) + ) + logger.info( + "Resolved robust policy from summary: " + f"risk_tolerance={policy['risk_tolerance']:.4f}, " + f"uncertainty_aversion={policy['uncertainty_aversion']:.4f}, " + f"min_budget_utilization={policy['min_budget_utilization']:.4f}, " + f"pd_cap_slack_penalty={policy['pd_cap_slack_penalty']:.4f}" + ) + return policy + + def _compute_realized_return( allocation: dict[int, float], loan_amnt: np.ndarray, @@ -90,141 +277,22 @@ def _resolve_robust_policy( champion_policy_path: str = "models/champion_portfolio_policy.json", ) -> dict[str, Any]: """Resolve robust strategy parameters from tradeoff summary, with fallback defaults.""" - default: dict[str, Any] = { - "source": "fallback_default", - "risk_tolerance": float(max_portfolio_pd), - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "hard_worst_case", - "gamma": 1.0, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - } champion_path = _artifact_path(champion_policy_path) - if champion_path.exists(): - try: - payload = json.loads(champion_path.read_text(encoding="utf-8")) - if isinstance(payload, dict): - if policy_selector == "robustness_aware": - selected = payload.get("selected_policy_robustness_aware") or payload.get( - "selected_policy", {} - ) - elif policy_selector == "balanced_robustness": - selected = ( - payload.get("selected_policy_balanced_robustness") - or payload.get("selected_policy_guardrail_robustness") - or payload.get("selected_policy_robustness_aware") - or payload.get("selected_policy", {}) - ) - elif policy_selector == "guardrail_robustness": - selected = ( - payload.get("selected_policy_guardrail_robustness") - or payload.get("selected_policy_balanced_robustness") - or payload.get("selected_policy_robustness_aware") - or payload.get("selected_policy", {}) - ) - elif policy_selector == "explicit_champion_only": - selected = payload.get("selected_policy", {}) - if not selected: - raise ValueError("Champion policy artifact missing selected_policy") - else: - selected = payload.get("selected_policy", {}) - else: - selected = {} - policy: dict[str, Any] = { - "source": f"champion_policy_artifact::{policy_selector}", - "risk_tolerance": float(selected.get("risk_tolerance", max_portfolio_pd)), - "uncertainty_aversion": float(selected.get("uncertainty_aversion", 0.0)), - "min_budget_utilization": float(selected.get("min_budget_utilization", 0.0)), - "pd_cap_slack_penalty": float(selected.get("pd_cap_slack_penalty", 0.0)), - "policy_mode": str(selected.get("policy_mode", "hard_worst_case")), - "gamma": float(selected.get("gamma", 1.0)), - "delta_cap_quantile": float(selected.get("delta_cap_quantile", 1.0)), - "tail_focus_quantile": float(selected.get("tail_focus_quantile", 1.0)), - } - logger.info( - "Resolved robust policy from champion artifact: " - f"risk_tolerance={policy['risk_tolerance']:.4f}, " - f"policy_mode={policy['policy_mode']}, gamma={policy['gamma']:.2f}" - ) - return policy - except Exception as exc: - if policy_selector == "explicit_champion_only": - raise - logger.warning( - f"Could not parse champion portfolio policy ({champion_path}): {exc}. " - "Falling back to summary-based policy." - ) - elif policy_selector == "explicit_champion_only": - raise FileNotFoundError(f"Missing champion portfolio policy artifact: {champion_path}") - - path = _artifact_path(summary_path) - if not path.exists(): - logger.warning(f"Robustness summary not found ({path}); using fallback robust policy.") - return default - - try: - summary = pd.read_parquet(path) - except Exception as exc: - logger.warning(f"Could not read robustness summary ({path}): {exc}") - return default - - required_cols = { - "risk_tolerance", - "best_robust_lambda", - "best_robust_min_budget_utilization", - "best_robust_pd_cap_slack_penalty", - } - if summary.empty or not required_cols.issubset(set(summary.columns)): - missing = sorted(required_cols - set(summary.columns)) - logger.warning( - "Robustness summary missing required columns or empty; " - f"missing={missing}. Using fallback robust policy." - ) - return default - - work = summary.copy() - for col in required_cols: - work[col] = pd.to_numeric(work[col], errors="coerce") - work = work.dropna(subset=list(required_cols)).reset_index(drop=True) - if work.empty: - logger.warning("No valid numeric robust summary rows; using fallback policy.") - return default - - target = float(max_portfolio_pd) - lower_eq = work.loc[work["risk_tolerance"] <= target + 1e-12].copy() - candidate_pool = lower_eq if not lower_eq.empty else work - candidate_pool["_distance"] = (candidate_pool["risk_tolerance"] - target).abs() - if "best_robust_return" in candidate_pool.columns: - candidate_pool["best_robust_return"] = pd.to_numeric( - candidate_pool["best_robust_return"], errors="coerce" - ).fillna(float("-inf")) - row = candidate_pool.sort_values( - by=["_distance", "best_robust_return"], - ascending=[True, False], - ).iloc[0] - else: - row = candidate_pool.sort_values(by=["_distance"], ascending=[True]).iloc[0] + champion_policy = _resolve_champion_robust_policy( + champion_path=champion_path, + max_portfolio_pd=max_portfolio_pd, + policy_selector=policy_selector, + ) + if champion_policy is not None: + return champion_policy - policy = { - "source": "portfolio_robustness_summary", - "risk_tolerance": float(row["risk_tolerance"]), - "uncertainty_aversion": float(row["best_robust_lambda"]), - "min_budget_utilization": float(row["best_robust_min_budget_utilization"]), - "pd_cap_slack_penalty": float(row["best_robust_pd_cap_slack_penalty"]), - "policy_mode": str(row.get("best_robust_policy_mode", "hard_worst_case")), - "gamma": float(row.get("best_robust_gamma", 1.0)), - "delta_cap_quantile": float(row.get("best_robust_delta_cap_quantile", 1.0)), - } - logger.info( - "Resolved robust policy from summary: " - f"risk_tolerance={policy['risk_tolerance']:.4f}, " - f"uncertainty_aversion={policy['uncertainty_aversion']:.4f}, " - f"min_budget_utilization={policy['min_budget_utilization']:.4f}, " - f"pd_cap_slack_penalty={policy['pd_cap_slack_penalty']:.4f}" + summary_policy = _resolve_summary_robust_policy( + path=_artifact_path(summary_path), + max_portfolio_pd=max_portfolio_pd, ) - return policy + if summary_policy is not None: + return summary_policy + return _default_robust_policy(max_portfolio_pd) def _apply_candidate_universe( @@ -439,60 +507,27 @@ def _run_strategy( robust_policy: dict[str, Any] | None = None, ) -> tuple[dict, np.ndarray]: pd_point = np.asarray(common["pd_point"], dtype=float) - pd_high = np.asarray(common["pd_high"], dtype=float) - if robust: - policy = robust_policy or {} - loans = cast(pd.DataFrame, common["loans"]) - segment_labels: np.ndarray | None = None - if str(policy.get("policy_mode", "hard_worst_case")) in { - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - }: - grade = ( - loans["grade"].fillna("unknown").astype(str) - if "grade" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - term = ( - loans["term"].fillna("unknown").astype(str) - if "term" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - verification = ( - loans["verification_status"].fillna("unknown").astype(str) - if "verification_status" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - segment_labels = (grade + "|" + term + "|" + verification).to_numpy(dtype=object) - effective_pd = compute_effective_pd( - pd_point=pd_point, - pd_high=pd_high, - policy_mode=str(policy.get("policy_mode", "hard_worst_case")), - gamma=float(policy.get("gamma", 1.0)), - delta_cap_quantile=float(policy.get("delta_cap_quantile", 1.0)), - tail_focus_quantile=float(policy.get("tail_focus_quantile", 1.0)), - segment_labels=segment_labels, - ) - solution = optimize_portfolio_allocation( - robust=True, - uncertainty_aversion=float(policy.get("uncertainty_aversion", 0.0)), - min_budget_utilization=float(policy.get("min_budget_utilization", 0.0)), - pd_cap_slack_penalty=float(policy.get("pd_cap_slack_penalty", 0.0)), - pd_constraint_override=effective_pd, - total_budget=total_budget, - max_portfolio_pd=max_portfolio_pd, - solver_backend=solver_backend, - **common, - ) - else: - solution = optimize_portfolio_allocation( - robust=False, - total_budget=total_budget, - max_portfolio_pd=max_portfolio_pd, - solver_backend=solver_backend, - **common, - ) - return solution, pd_point + policy = robust_policy or {} + result = solve_policy_allocation( + loans=cast(pd.DataFrame, common["loans"]), + pd_point=pd_point, + pd_low=np.asarray(common["pd_low"], dtype=float), + pd_high=np.asarray(common["pd_high"], dtype=float), + lgd=np.asarray(common["lgd"], dtype=float), + int_rates=np.asarray(common["int_rates"], dtype=float), + total_budget=total_budget, + risk_tolerance=max_portfolio_pd, + robust=robust, + uncertainty_aversion=float(policy.get("uncertainty_aversion", 0.0)), + min_budget_utilization=float(policy.get("min_budget_utilization", 0.0)), + pd_cap_slack_penalty=float(policy.get("pd_cap_slack_penalty", 0.0)), + policy_mode=str(policy.get("policy_mode", "hard_worst_case")), + gamma=float(policy.get("gamma", 1.0)), + delta_cap_quantile=float(policy.get("delta_cap_quantile", 1.0)), + tail_focus_quantile=float(policy.get("tail_focus_quantile", 1.0)), + solver_backend=solver_backend, + ) + return result.solution, result.effective_pd def _candidate_metrics( diff --git a/scripts/train_pd_model.py b/scripts/train_pd_model.py index c133740..8dc23ae 100644 --- a/scripts/train_pd_model.py +++ b/scripts/train_pd_model.py @@ -10,6 +10,7 @@ import json import pickle import shutil +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -70,6 +71,58 @@ pd.DataFrame, pd.Series, ] +REQUIRED_CONFIG_SECTIONS = ("output", "feature_source", "data", "hpo", "validation") +REQUIRED_DATA_KEYS = ("train_path", "test_path", "calibration_path") +OPTIONAL_MAPPING_SECTIONS = ( + "output", + "feature_source", + "data", + "hpo", + "validation", + "model", + "conformal", + "calibration", +) +OUTPUT_DEFAULTS = { + "model_path": "models/pd_canonical.cbm", + "default_model_path": "models/pd_catboost_default.cbm", + "tuned_model_path": "models/pd_canonical.cbm", + "conformal_path": "models/pd_canonical_calibrator.pkl", + "status_path": "models/pd_training_status.json", + "checkpoint_dir": "models/pd_training_checkpoints", + "brier_decomposition_path": "data/processed/brier_score_decomposition.json", + "murphy_diagram_path": "reports/figures/calibration/murphy_diagram.png", + "canonical_model_path": str(CANONICAL_MODEL_PATH), + "canonical_calibrator_path": str(CANONICAL_CALIBRATOR_PATH), + "contract_path": str(CONTRACT_PATH), + "logreg_model_path": "models/pd_logreg_baseline.pkl", + "training_record_path": "models/pd_training_record.pkl", + "seed_replay_status_path": "models/pd_hpo_seed_replay_status.json", + "test_predictions_path": "data/processed/test_predictions.parquet", + "shap_dir": "reports/figures/shap", + "threshold_semantics_path": "models/threshold_semantics.json", +} + + +def _metric_float(metrics: Mapping[str, Any], key: str, default: float = 0.0) -> float: + value = metrics.get(key, default) + if value is None or isinstance(value, Mapping): + return float(default) + return float(value) + + +def _metric_int(metrics: Mapping[str, Any], key: str, default: int = 0) -> int: + value = metrics.get(key, default) + if value is None or isinstance(value, Mapping): + return int(default) + return int(value) + + +def _metric_mapping( + metrics: Mapping[str, Any], key: str, default: Mapping[str, Any] +) -> dict[str, Any]: + value = metrics.get(key, default) + return dict(value) if isinstance(value, Mapping) else dict(default) def load_config(config_path: str) -> dict[str, Any]: @@ -77,112 +130,93 @@ def load_config(config_path: str) -> dict[str, Any]: return yaml.safe_load(f) +def _missing_mapping_keys(config: Mapping[str, Any], keys: tuple[str, ...]) -> list[str]: + return [key for key in keys if key not in config] + + +def _missing_required_data_keys(config: Mapping[str, Any]) -> list[str]: + data_cfg = config.get("data", {}) or {} + return [key for key in REQUIRED_DATA_KEYS if not str(data_cfg.get(key, "")).strip()] + + +def _normalize_config_sections(config: Mapping[str, Any]) -> dict[str, Any]: + normalized = dict(config) + for section in OPTIONAL_MAPPING_SECTIONS: + normalized[section] = dict(config.get(section, {}) or {}) + return normalized + + +def _apply_pd_config_defaults(config: dict[str, Any]) -> dict[str, Any]: + normalized = dict(config) + output_cfg = dict(normalized.get("output", {}) or {}) + for key, value in OUTPUT_DEFAULTS.items(): + output_cfg.setdefault(key, value) + normalized["output"] = output_cfg + feature_source_cfg = dict(normalized.get("feature_source", {}) or {}) + feature_source_cfg.setdefault("mode", "auto") + feature_source_cfg.setdefault("feature_config_path", "data/processed/feature_config.yml") + normalized["feature_source"] = feature_source_cfg + return normalized + + def validate_pd_config(config: dict[str, Any], *, config_path: str) -> dict[str, Any]: """Fail fast on missing PD config sections and required keys.""" if not isinstance(config, dict): raise ValueError(f"PD config must be a mapping: {config_path}") - required_sections = ("output", "feature_source", "data", "hpo", "validation") - missing_sections = [section for section in required_sections if section not in config] + missing_sections = _missing_mapping_keys(config, REQUIRED_CONFIG_SECTIONS) if missing_sections: raise ValueError( f"PD config {config_path} missing required sections: {', '.join(missing_sections)}" ) - required_data_keys = ("train_path", "test_path", "calibration_path") - missing_data_keys = [ - key - for key in required_data_keys - if not str((config.get("data", {}) or {}).get(key, "")).strip() - ] + missing_data_keys = _missing_required_data_keys(config) if missing_data_keys: raise ValueError( f"PD config {config_path} missing required data keys: {', '.join(missing_data_keys)}" ) - normalized = dict(config) - normalized["output"] = dict(config.get("output", {}) or {}) - normalized["feature_source"] = dict(config.get("feature_source", {}) or {}) - normalized["data"] = dict(config.get("data", {}) or {}) - normalized["hpo"] = dict(config.get("hpo", {}) or {}) - normalized["validation"] = dict(config.get("validation", {}) or {}) - normalized["model"] = dict(config.get("model", {}) or {}) - normalized["conformal"] = dict(config.get("conformal", {}) or {}) - normalized["calibration"] = dict(config.get("calibration", {}) or {}) - - output_defaults = { - "model_path": "models/pd_canonical.cbm", - "default_model_path": "models/pd_catboost_default.cbm", - "tuned_model_path": "models/pd_canonical.cbm", - "conformal_path": "models/pd_canonical_calibrator.pkl", - "status_path": "models/pd_training_status.json", - "checkpoint_dir": "models/pd_training_checkpoints", - "brier_decomposition_path": "data/processed/brier_score_decomposition.json", - "murphy_diagram_path": "reports/figures/calibration/murphy_diagram.png", - "canonical_model_path": str(CANONICAL_MODEL_PATH), - "canonical_calibrator_path": str(CANONICAL_CALIBRATOR_PATH), - "contract_path": str(CONTRACT_PATH), - "logreg_model_path": "models/pd_logreg_baseline.pkl", - "training_record_path": "models/pd_training_record.pkl", - "seed_replay_status_path": "models/pd_hpo_seed_replay_status.json", - "test_predictions_path": "data/processed/test_predictions.parquet", - "shap_dir": "reports/figures/shap", - "threshold_semantics_path": "models/threshold_semantics.json", - } - for key, value in output_defaults.items(): - normalized["output"].setdefault(key, value) - - normalized["feature_source"].setdefault("mode", "auto") - normalized["feature_source"].setdefault( - "feature_config_path", "data/processed/feature_config.yml" - ) - return normalized + return _apply_pd_config_defaults(_normalize_config_sections(config)) -def _apply_cli_overrides( - config: dict[str, Any], +def _override_training_regime( + config: Mapping[str, Any], *, - training_regime_mode: str | None = None, - recent_window_quarters: int | None = None, - half_life_quarters: int | None = None, - stable_core_enabled: bool | None = None, - hpo_n_trials: int | None = None, - hpo_enabled: bool | None = None, - challenger_enabled: bool | None = None, - walk_forward_enabled: bool | None = None, - seed_replay_enabled: bool | None = None, - catboost_iterations: int | None = None, + training_regime_mode: str | None, + recent_window_quarters: int | None, + half_life_quarters: int | None, ) -> dict[str, Any]: - """Return a config copy with command-line overrides applied.""" - updated = dict(config) - - regime_cfg = dict(updated.get("training_regime", {}) or {}) + regime_cfg = dict(config.get("training_regime", {}) or {}) if training_regime_mode is not None: regime_cfg["mode"] = str(training_regime_mode) if recent_window_quarters is not None: regime_cfg["recent_window_quarters"] = int(recent_window_quarters) if half_life_quarters is not None: regime_cfg["half_life_quarters"] = int(half_life_quarters) - updated["training_regime"] = regime_cfg + return regime_cfg - stable_core_cfg = dict(updated.get("stable_core", {}) or {}) - if stable_core_enabled is not None: - stable_core_cfg["enabled"] = bool(stable_core_enabled) - updated["stable_core"] = stable_core_cfg - hpo_cfg = dict(updated.get("hpo", {}) or {}) +def _override_hpo_config( + config: Mapping[str, Any], + *, + hpo_n_trials: int | None, + hpo_enabled: bool | None, +) -> dict[str, Any]: + hpo_cfg = dict(config.get("hpo", {}) or {}) if hpo_n_trials is not None: hpo_cfg["n_trials"] = int(hpo_n_trials) if hpo_enabled is not None: hpo_cfg["enabled"] = bool(hpo_enabled) - updated["hpo"] = hpo_cfg + return hpo_cfg - challenger_cfg = dict(updated.get("challenger_pipeline", {}) or {}) - if challenger_enabled is not None: - challenger_cfg["enabled"] = bool(challenger_enabled) - updated["challenger_pipeline"] = challenger_cfg - validation_cfg = dict(updated.get("validation", {}) or {}) +def _override_validation_config( + config: Mapping[str, Any], + *, + walk_forward_enabled: bool | None, + seed_replay_enabled: bool | None, +) -> dict[str, Any]: + validation_cfg = dict(config.get("validation", {}) or {}) walk_cfg = dict(validation_cfg.get("walk_forward", {}) or {}) if walk_forward_enabled is not None: walk_cfg["enabled"] = bool(walk_forward_enabled) @@ -191,14 +225,67 @@ def _apply_cli_overrides( if seed_replay_enabled is not None: seed_cfg["enabled"] = bool(seed_replay_enabled) validation_cfg["seed_replay"] = seed_cfg - updated["validation"] = validation_cfg + return validation_cfg - model_cfg = dict(updated.get("model", {}) or {}) + +def _override_model_config( + config: Mapping[str, Any], + *, + catboost_iterations: int | None, +) -> dict[str, Any]: + model_cfg = dict(config.get("model", {}) or {}) model_params = dict(model_cfg.get("params", {}) or {}) if catboost_iterations is not None: model_params["iterations"] = int(catboost_iterations) model_cfg["params"] = model_params - updated["model"] = model_cfg + return model_cfg + + +def _apply_cli_overrides( + config: dict[str, Any], + *, + training_regime_mode: str | None = None, + recent_window_quarters: int | None = None, + half_life_quarters: int | None = None, + stable_core_enabled: bool | None = None, + hpo_n_trials: int | None = None, + hpo_enabled: bool | None = None, + challenger_enabled: bool | None = None, + walk_forward_enabled: bool | None = None, + seed_replay_enabled: bool | None = None, + catboost_iterations: int | None = None, +) -> dict[str, Any]: + """Return a config copy with command-line overrides applied.""" + updated = dict(config) + updated["training_regime"] = _override_training_regime( + updated, + training_regime_mode=training_regime_mode, + recent_window_quarters=recent_window_quarters, + half_life_quarters=half_life_quarters, + ) + + stable_core_cfg = dict(updated.get("stable_core", {}) or {}) + if stable_core_enabled is not None: + stable_core_cfg["enabled"] = bool(stable_core_enabled) + updated["stable_core"] = stable_core_cfg + + updated["hpo"] = _override_hpo_config( + updated, + hpo_n_trials=hpo_n_trials, + hpo_enabled=hpo_enabled, + ) + + challenger_cfg = dict(updated.get("challenger_pipeline", {}) or {}) + if challenger_enabled is not None: + challenger_cfg["enabled"] = bool(challenger_enabled) + updated["challenger_pipeline"] = challenger_cfg + + updated["validation"] = _override_validation_config( + updated, + walk_forward_enabled=walk_forward_enabled, + seed_replay_enabled=seed_replay_enabled, + ) + updated["model"] = _override_model_config(updated, catboost_iterations=catboost_iterations) return updated @@ -278,22 +365,24 @@ def _metric_with_aliases(payload: dict[str, Any], *keys: str) -> float | None: return None -def _validate_replay_expectations( - *, - replay_cfg: dict[str, Any], - final_test_metrics: dict[str, Any], - feature_names: list[str], - config_path: str, -) -> None: - expected = dict(replay_cfg.get("expectations") or {}) - tolerances = dict(replay_cfg.get("tolerances") or {}) +def _validate_replay_feature_order(replay_cfg: Mapping[str, Any], feature_names: list[str]) -> None: expected_features = [str(x) for x in replay_cfg.get("feature_names", [])] if expected_features and list(feature_names) != expected_features: raise ValueError("Replay feature order mismatch against frozen manifest.") + + +def _validate_replay_config_path(replay_cfg: Mapping[str, Any], config_path: str) -> None: manifest_config_path = str(replay_cfg.get("config_path", "")).strip() if manifest_config_path and manifest_config_path != str(config_path): raise ValueError("Replay config_path does not match frozen manifest.") + +def _replay_metric_violations( + *, + expected: dict[str, Any], + tolerances: dict[str, Any], + observed_metrics: dict[str, Any], +) -> list[str]: aliases = { "auc_roc": ("auc_roc",), "brier_score": ("brier_score",), @@ -304,13 +393,32 @@ def _validate_replay_expectations( for name, key_aliases in aliases.items(): expected_value = _metric_with_aliases(expected, name, *key_aliases) tolerance = _metric_with_aliases(tolerances, name, *key_aliases) - observed = _metric_with_aliases(final_test_metrics, name, *key_aliases) + observed = _metric_with_aliases(observed_metrics, name, *key_aliases) if expected_value is None or tolerance is None or observed is None: continue if abs(observed - expected_value) > tolerance: violations.append( f"{name}: observed={observed:.6f} expected={expected_value:.6f} tol={tolerance:.6f}" ) + return violations + + +def _validate_replay_expectations( + *, + replay_cfg: dict[str, Any], + final_test_metrics: dict[str, Any], + feature_names: list[str], + config_path: str, +) -> None: + expected = dict(replay_cfg.get("expectations") or {}) + tolerances = dict(replay_cfg.get("tolerances") or {}) + _validate_replay_feature_order(replay_cfg, feature_names) + _validate_replay_config_path(replay_cfg, config_path) + violations = _replay_metric_violations( + expected=expected, + tolerances=tolerances, + observed_metrics=final_test_metrics, + ) if violations: raise ValueError("Replay metric validation failed: " + "; ".join(violations)) @@ -439,6 +547,61 @@ def _apply_stable_core( ) +def _feature_source_config(config: dict[str, Any]) -> tuple[str, str | Path]: + feature_src_cfg = dict(config.get("feature_source", {}) or {}) + feature_mode = str(feature_src_cfg.get("mode", "auto")) + feature_config_path = feature_src_cfg.get( + "feature_config_path", "data/processed/feature_config.yml" + ) + return feature_mode, feature_config_path + + +def _resolved_feature_lists( + feature_sets: Mapping[str, Any], +) -> tuple[list[str], list[str], list[str]]: + return ( + [str(x) for x in feature_sets["catboost_features"]], + [str(x) for x in feature_sets["logreg_features"]], + [str(x) for x in feature_sets["categorical_features"]], + ) + + +def _apply_replay_feature_override( + *, + catboost_features: list[str], + categorical_features: list[str], + run_mode: str, + replay_cfg: dict[str, Any], +) -> tuple[list[str], list[str]]: + if run_mode != "replay": + return catboost_features, categorical_features + replay_features = [str(x) for x in replay_cfg.get("feature_names", [])] + if not replay_features: + return catboost_features, categorical_features + replay_categorical = [str(x) for x in replay_cfg.get("categorical_features", [])] + return replay_features, replay_categorical + + +def _features_in_all_splits( + features: list[str], + train: pd.DataFrame, + cal: pd.DataFrame, + test: pd.DataFrame, +) -> list[str]: + return [c for c in features if c in train.columns and c in cal.columns and c in test.columns] + + +def _validate_resolved_training_features( + *, + catboost_features: list[str], + logreg_features: list[str], +) -> None: + if not catboost_features: + raise ValueError("No CatBoost features resolved across train/cal/test splits.") + if not logreg_features: + raise ValueError("No Logistic Regression features resolved across train/cal/test splits.") + + def _resolve_training_features( *, config: dict[str, Any], @@ -448,36 +611,21 @@ def _resolve_training_features( run_mode: str, replay_cfg: dict[str, Any], ) -> ResolvedFeatureTuple: - feature_src_cfg = dict(config.get("feature_source", {}) or {}) - feature_mode = str(feature_src_cfg.get("mode", "auto")) - feature_config_path = feature_src_cfg.get( - "feature_config_path", "data/processed/feature_config.yml" - ) - + feature_mode, feature_config_path = _feature_source_config(config) feature_sets = resolve_feature_sets( train, feature_source=feature_mode, feature_config_path=feature_config_path, ) - catboost_features = list(feature_sets["catboost_features"]) - logreg_features = list(feature_sets["logreg_features"]) - categorical_features = list(feature_sets["categorical_features"]) - - if run_mode == "replay": - replay_features = [str(x) for x in replay_cfg.get("feature_names", [])] - replay_categorical = [str(x) for x in replay_cfg.get("categorical_features", [])] - if replay_features: - catboost_features = replay_features - categorical_features = replay_categorical - - catboost_features = [ - c - for c in catboost_features - if c in train.columns and c in cal.columns and c in test.columns - ] - logreg_features = [ - c for c in logreg_features if c in train.columns and c in cal.columns and c in test.columns - ] + catboost_features, logreg_features, categorical_features = _resolved_feature_lists(feature_sets) + catboost_features, categorical_features = _apply_replay_feature_override( + catboost_features=catboost_features, + categorical_features=categorical_features, + run_mode=run_mode, + replay_cfg=replay_cfg, + ) + catboost_features = _features_in_all_splits(catboost_features, train, cal, test) + logreg_features = _features_in_all_splits(logreg_features, train, cal, test) categorical_features = [c for c in categorical_features if c in catboost_features] catboost_features, categorical_features, stable_core_meta = _apply_stable_core( @@ -486,11 +634,10 @@ def _resolve_training_features( {} if run_mode == "replay" else (config.get("stable_core", {}) or {}), ) logreg_features = [c for c in logreg_features if c in catboost_features] - - if not catboost_features: - raise ValueError("No CatBoost features resolved across train/cal/test splits.") - if not logreg_features: - raise ValueError("No Logistic Regression features resolved across train/cal/test splits.") + _validate_resolved_training_features( + catboost_features=catboost_features, + logreg_features=logreg_features, + ) return ( str(feature_sets.get("feature_source", feature_mode)), @@ -797,6 +944,132 @@ def _fairness_row( } +def _decision_threshold_metadata(run_tag: str) -> dict[str, Any]: + return build_artifact_metadata( + schema_version="2026-03-01.1", + run_tag=run_tag, + require_explicit=True, + ) + + +def _fallback_decision_threshold_artifact( + *, + config: dict[str, Any], + run_tag: str, +) -> dict[str, Any]: + return { + "enabled": False, + "selected_threshold": float(config.get("calibration", {}).get("default_threshold", 0.5)), + "source": "fallback_default", + **_decision_threshold_metadata(run_tag), + } + + +def _replay_decision_threshold_artifact( + *, + replay_cfg: dict[str, Any], + run_tag: str, +) -> dict[str, Any]: + artifact = dict(replay_cfg["decision_threshold_artifact"]) + artifact.update(_decision_threshold_metadata(run_tag)) + artifact["source"] = "frozen_replay_manifest" + return artifact + + +def _load_fairness_threshold_policy(fairness_policy_path: str) -> tuple[dict[str, Any], list[Any]]: + with open(fairness_policy_path, encoding="utf-8") as f: + fairness_cfg = yaml.safe_load(f) or {} + return dict(fairness_cfg.get("policy", {}) or {}), list( + fairness_cfg.get("attributes", []) or [] + ) + + +def _threshold_grid(decision_cfg: dict[str, Any]) -> np.ndarray: + threshold_min = float(decision_cfg.get("min_threshold", 0.05)) + threshold_max = float(decision_cfg.get("max_threshold", 0.95)) + threshold_step = float(decision_cfg.get("step", 0.01)) + return np.arange(threshold_min, threshold_max + (threshold_step / 2.0), threshold_step) + + +def _search_decision_threshold_artifact( + *, + decision_cfg: dict[str, Any], + train_val: pd.DataFrame, + cal: pd.DataFrame, + y_val: pd.Series, + y_prob_final_val: np.ndarray, + y_cal: pd.Series, + y_prob_tuned_cal: np.ndarray, + selected_cal_method: str, + run_tag: str, +) -> dict[str, Any]: + fairness_policy_path = str( + decision_cfg.get("fairness_policy_path", "configs/fairness_policy.yaml") + ) + fairness_policy, fairness_attrs = _load_fairness_threshold_policy(fairness_policy_path) + fallback_threshold = float(fairness_policy.get("prediction_threshold", 0.5)) + threshold_result = _select_decision_threshold( + y_true=y_val.to_numpy(), + y_prob=y_prob_final_val, + policy={ + "dpd_threshold": float(fairness_policy.get("dpd_threshold", 0.10)), + "eo_gap_threshold": float(fairness_policy.get("eo_gap_threshold", 0.10)), + "dir_threshold": float(fairness_policy.get("dir_threshold", 0.80)), + }, + groups_dict=_build_fairness_groups_for_threshold(train_val, fairness_attrs), + thresholds=_threshold_grid(decision_cfg), + fallback_threshold=fallback_threshold, + y_true_secondary=y_cal.to_numpy(), + y_prob_secondary=y_prob_tuned_cal, + groups_dict_secondary=_build_fairness_groups_for_threshold(cal, fairness_attrs), + ) + return { + "enabled": True, + "selected_threshold": float(threshold_result["selected_threshold"]), + "fallback_threshold": fallback_threshold, + "selection_metrics": threshold_result["selection_metrics"], + "search_summary": threshold_result["search_summary"], + "source": "validation_fairness_search", + "fairness_policy_path": fairness_policy_path, + "validation_rows": len(train_val), + "secondary_validation_rows": len(cal), + "calibration_method": selected_cal_method, + **_decision_threshold_metadata(run_tag), + } + + +def _resolve_decision_threshold_artifact( + *, + config: dict[str, Any], + run_mode: str, + replay_cfg: dict[str, Any], + decision_cfg: dict[str, Any], + train_val: pd.DataFrame, + cal: pd.DataFrame, + y_val: pd.Series, + y_prob_final_val: np.ndarray, + y_cal: pd.Series, + y_prob_tuned_cal: np.ndarray, + selected_cal_method: str, + run_tag: str, +) -> dict[str, Any]: + if run_mode == "replay" and replay_cfg.get("decision_threshold_artifact"): + return _replay_decision_threshold_artifact(replay_cfg=replay_cfg, run_tag=run_tag) + if bool(decision_cfg.get("enabled", True)): + return _search_decision_threshold_artifact( + decision_cfg=decision_cfg, + train_val=train_val, + cal=cal, + y_val=y_val, + y_prob_final_val=y_prob_final_val, + y_cal=y_cal, + y_prob_tuned_cal=y_prob_tuned_cal, + selected_cal_method=selected_cal_method, + run_tag=run_tag, + ) + return _fallback_decision_threshold_artifact(config=config, run_tag=run_tag) + + def _build_calibration_backtest_splits( cal_df: pd.DataFrame, n_folds: int = 4, @@ -829,73 +1102,68 @@ def _build_calibration_backtest_splits( return splits -def _evaluate_calibration_method( +def _empty_calibration_method_report(method: str) -> dict[str, Any]: + return { + "method": method, + "folds_used": 0, + "mean_brier": float("inf"), + "mean_log_loss": float("inf"), + "mean_ece": float("inf"), + "mean_auc_drop": float("inf"), + "brier_variance": float("inf"), + "ece_variance": float("inf"), + "stability": float("inf"), + "degradation_rate": 1.0, + "folds": [], + } + + +def _evaluate_calibration_fold( + *, method: str, + fold_id: int, + idx_fit: np.ndarray, + idx_eval: np.ndarray, y_true: np.ndarray, y_prob_raw: np.ndarray, - splits: list[tuple[np.ndarray, np.ndarray]], -) -> dict[str, Any]: - """Backtest calibrator over temporal folds using multi-metric summary.""" - fold_rows: list[dict[str, Any]] = [] - - for fold_id, (idx_fit, idx_eval) in enumerate(splits, start=1): - y_fit = y_true[idx_fit] - y_eval = y_true[idx_eval] - p_fit = y_prob_raw[idx_fit] - p_eval = y_prob_raw[idx_eval] +) -> dict[str, Any] | None: + y_fit = y_true[idx_fit] + y_eval = y_true[idx_eval] + p_fit = y_prob_raw[idx_fit] + p_eval = y_prob_raw[idx_eval] + if len(np.unique(y_fit)) < 2 or len(np.unique(y_eval)) < 2: + return None - if len(np.unique(y_fit)) < 2 or len(np.unique(y_eval)) < 2: - continue + calibrator = _fit_calibrator_from_scores(method, y_fit, p_fit) + p_eval_cal = _apply_calibrator(calibrator, p_eval) + raw_auc = _safe_auc(y_eval, p_eval) + cal_auc = _safe_auc(y_eval, p_eval_cal) + auc_drop = 0.0 if raw_auc is None or cal_auc is None else float(raw_auc - cal_auc) + brier_raw = float(brier_score_loss(y_eval, p_eval)) + brier_cal = float(brier_score_loss(y_eval, p_eval_cal)) + return { + "fold": fold_id, + "n_fit": len(idx_fit), + "n_eval": len(idx_eval), + "raw_auc": None if raw_auc is None else float(raw_auc), + "cal_auc": None if cal_auc is None else float(cal_auc), + "auc_drop": float(auc_drop), + "brier": brier_cal, + "brier_raw": brier_raw, + "brier_degraded": brier_cal > brier_raw, + "log_loss": float(log_loss(y_eval, p_eval_cal)), + "ece": float(expected_calibration_error(y_eval, p_eval_cal)), + } - calibrator = _fit_calibrator_from_scores(method, y_fit, p_fit) - p_eval_cal = _apply_calibrator(calibrator, p_eval) - - raw_auc = _safe_auc(y_eval, p_eval) - cal_auc = _safe_auc(y_eval, p_eval_cal) - auc_drop = 0.0 - if raw_auc is not None and cal_auc is not None: - auc_drop = float(raw_auc - cal_auc) - - brier_raw = float(brier_score_loss(y_eval, p_eval)) - brier_cal = float(brier_score_loss(y_eval, p_eval_cal)) - - fold_rows.append( - { - "fold": fold_id, - "n_fit": len(idx_fit), - "n_eval": len(idx_eval), - "raw_auc": None if raw_auc is None else float(raw_auc), - "cal_auc": None if cal_auc is None else float(cal_auc), - "auc_drop": float(auc_drop), - "brier": brier_cal, - "brier_raw": brier_raw, - "brier_degraded": brier_cal > brier_raw, - "log_loss": float(log_loss(y_eval, p_eval_cal)), - "ece": float(expected_calibration_error(y_eval, p_eval_cal)), - } - ) +def _summarize_calibration_folds(method: str, fold_rows: list[dict[str, Any]]) -> dict[str, Any]: if not fold_rows: - return { - "method": method, - "folds_used": 0, - "mean_brier": float("inf"), - "mean_log_loss": float("inf"), - "mean_ece": float("inf"), - "mean_auc_drop": float("inf"), - "brier_variance": float("inf"), - "ece_variance": float("inf"), - "stability": float("inf"), - "degradation_rate": 1.0, - "folds": [], - } - - briers = np.array([r["brier"] for r in fold_rows], dtype=float) - log_losses = np.array([r["log_loss"] for r in fold_rows], dtype=float) - eces = np.array([r["ece"] for r in fold_rows], dtype=float) - auc_drops = np.array([r["auc_drop"] for r in fold_rows], dtype=float) - n_degraded = sum(1 for r in fold_rows if r.get("brier_degraded", False)) - + return _empty_calibration_method_report(method) + briers = np.array([row["brier"] for row in fold_rows], dtype=float) + log_losses = np.array([row["log_loss"] for row in fold_rows], dtype=float) + eces = np.array([row["ece"] for row in fold_rows], dtype=float) + auc_drops = np.array([row["auc_drop"] for row in fold_rows], dtype=float) + n_degraded = sum(1 for row in fold_rows if row.get("brier_degraded", False)) return { "method": method, "folds_used": len(fold_rows), @@ -911,6 +1179,121 @@ def _evaluate_calibration_method( } +def _write_calibration_cumulative_diffs( + statistical_cal_tests: dict[str, object], + *, + n_rows: int, +) -> None: + if "_cum_diff_calibrated" not in statistical_cal_tests: + return + k_idx = np.arange(n_rows) / n_rows + sigma_raw = statistical_cal_tests.get("_sigma", 0.0) + sigma_val = float(sigma_raw) if isinstance(sigma_raw, str | int | float | np.number) else 0.0 + cum_diff_df = pd.DataFrame( + { + "k": k_idx, + "cum_diff_calibrated": statistical_cal_tests.pop("_cum_diff_calibrated"), + "cum_diff_uncalibrated": statistical_cal_tests.pop( + "_cum_diff_uncalibrated", + [float("nan")] * len(k_idx), + ), + "sigma_upper": sigma_val * 2, + "sigma_lower": -sigma_val * 2, + } + ) + cum_diff_path = _artifact_path("data/processed/calibration_cumulative_diffs.parquet") + cum_diff_path.parent.mkdir(parents=True, exist_ok=True) + cum_diff_df.to_parquet(cum_diff_path, index=False) + logger.info(f"Saved calibration cumulative diffs: {cum_diff_path}") + statistical_cal_tests.pop("_sigma", None) + + +def _write_statistical_calibration_json(statistical_cal_tests: dict[str, object]) -> None: + stat_cal_path = _artifact_path("data/processed/statistical_calibration_tests.json") + stat_cal_path.parent.mkdir(parents=True, exist_ok=True) + stat_cal_path.write_text( + json.dumps(statistical_cal_tests, indent=2, default=str), + encoding="utf-8", + ) + logger.info(f"Saved statistical calibration tests: {stat_cal_path}") + + +def _run_statistical_calibration_tests( + *, + y_test: pd.Series, + y_prob_final: np.ndarray, + y_prob_tuned_test: np.ndarray, +) -> None: + try: + from mapie.metrics.calibration import ( + cumulative_differences, + kolmogorov_smirnov_p_value, + kuiper_p_value, + length_scale, + spiegelhalter_p_value, + ) + + y_test_arr = y_test.values.astype(float) + statistical_cal_tests: dict[str, object] = {} + for tag, probs in [("calibrated", y_prob_final), ("uncalibrated", y_prob_tuned_test)]: + try: + ks_p = float(kolmogorov_smirnov_p_value(y_test_arr, probs)) + ku_p = float(kuiper_p_value(y_test_arr, probs)) + sp_p = float(spiegelhalter_p_value(y_test_arr, probs)) + cum_diff = cumulative_differences(y_test_arr, probs) + sigma = float(length_scale(probs)) + statistical_cal_tests[tag] = { + "ks_pvalue": ks_p, + "kuiper_pvalue": ku_p, + "spiegelhalter_pvalue": sp_p, + "length_scale_sigma": sigma, + "n": len(y_test_arr), + } + logger.info( + f"Calibration tests [{tag}]: KS_p={ks_p:.4f} " + f"Kuiper_p={ku_p:.4f} Spiegelhalter_p={sp_p:.4f}" + ) + if tag == "calibrated": + statistical_cal_tests["_cum_diff_calibrated"] = cum_diff.tolist() + statistical_cal_tests["_sigma"] = sigma + elif tag == "uncalibrated": + statistical_cal_tests["_cum_diff_uncalibrated"] = cum_diff.tolist() + except Exception as exc_inner: + logger.warning(f"Statistical calibration tests [{tag}] failed: {exc_inner}") + statistical_cal_tests[tag] = {"error": str(exc_inner)} + + _write_calibration_cumulative_diffs(statistical_cal_tests, n_rows=len(y_test_arr)) + _write_statistical_calibration_json(statistical_cal_tests) + except ImportError: + logger.warning( + "mapie.metrics.calibration not available - statistical calibration tests skipped." + ) + except Exception as exc: + logger.warning(f"Statistical calibration tests block failed: {exc}") + + +def _evaluate_calibration_method( + method: str, + y_true: np.ndarray, + y_prob_raw: np.ndarray, + splits: list[tuple[np.ndarray, np.ndarray]], +) -> dict[str, Any]: + """Backtest calibrator over temporal folds using multi-metric summary.""" + fold_rows: list[dict[str, Any]] = [] + for fold_id, (idx_fit, idx_eval) in enumerate(splits, start=1): + fold_row = _evaluate_calibration_fold( + method=method, + fold_id=fold_id, + idx_fit=idx_fit, + idx_eval=idx_eval, + y_true=y_true, + y_prob_raw=y_prob_raw, + ) + if fold_row is not None: + fold_rows.append(fold_row) + return _summarize_calibration_folds(method, fold_rows) + + def _select_calibration_method( reports: list[dict[str, Any]], auc_drop_limit: float = 0.0015, @@ -1177,181 +1560,636 @@ def _evaluate_walk_forward_auc( } -def _replay_top_optuna_trials( +def _walk_forward_disabled_report(walk_cfg: Mapping[str, Any]) -> dict[str, Any]: + return { + "enabled": False, + "reason": "disabled_in_config", + "n_windows_requested": int(walk_cfg.get("n_windows", 0)), + "n_windows_used": 0, + "folds": [], + } + + +def _evaluate_walk_forward_stage( *, - hpo_cfg: dict[str, Any], - base_params: dict[str, Any], - X_train_fit_cb: pd.DataFrame, - y_train_fit: pd.Series, - X_val_cb: pd.DataFrame, - y_val: pd.Series, - cat_features: list[str], - seeds: list[int], - top_k_trials: int = 3, - prioritize_gate_pass: bool = True, - sample_weight: np.ndarray | None = None, - eval_sample_weight: np.ndarray | None = None, + enabled: bool, + walk_cfg: Mapping[str, Any], + train: pd.DataFrame, + catboost_features: list[str], + categorical_features: list[str], + model_params: Mapping[str, Any], ) -> dict[str, Any]: - """Replay top Optuna trials across multiple seeds for robustness.""" - report: dict[str, Any] = { + if not enabled: + return _walk_forward_disabled_report(walk_cfg) + max_rows = int(walk_cfg.get("max_rows", 0)) + return _evaluate_walk_forward_auc( + train, + features=catboost_features, + categorical_features=categorical_features, + target=TARGET, + params=dict(model_params), + n_windows=int(walk_cfg.get("n_windows", 3)), + min_train_rows=int(walk_cfg.get("min_train_rows", 200_000)), + window_rows=int(walk_cfg.get("window_rows", 80_000)), + date_col=str(walk_cfg.get("date_col", "issue_d")), + max_rows=None if max_rows <= 0 else max_rows, + ) + + +def _export_shap_feature_importance( + *, + cb_tuned_model: Any, + X_test_cb: pd.DataFrame, + categorical_features: list[str], + catboost_features: list[str], + shap_dir: Path, +) -> dict[str, Any]: + shap_artifact: dict[str, Any] = {"exported": False} + try: + from catboost import Pool as _SHAPPool + + shap_pool = _SHAPPool(X_test_cb, cat_features=categorical_features) + shap_raw = cb_tuned_model.get_feature_importance(type="ShapValues", data=shap_pool) + # ShapValues returns (n_samples, n_features + 1); last col = expected value. + shap_values = shap_raw[:, :-1] + shap_expected = float(shap_raw[0, -1]) + + mean_abs_shap = np.abs(shap_values).mean(axis=0) + shap_importance = sorted( + zip(catboost_features, mean_abs_shap.tolist(), strict=False), + key=lambda x: x[1], + reverse=True, + ) + shap_dir.mkdir(parents=True, exist_ok=True) + + np.savez_compressed( + str(shap_dir / "shap_values_test.npz"), + shap_values=shap_values, + expected_value=np.array([shap_expected]), + feature_names=np.array(catboost_features), + ) + + top_n = min(20, len(shap_importance)) + shap_summary = { + "expected_value": shap_expected, + "n_samples": int(shap_values.shape[0]), + "n_features": int(shap_values.shape[1]), + "top_features": [ + {"feature": f, "mean_abs_shap": round(v, 6)} for f, v in shap_importance[:top_n] + ], + } + shap_summary_path = shap_dir / "shap_feature_importance.json" + shap_summary_path.write_text(json.dumps(shap_summary, indent=2), encoding="utf-8") + shap_artifact = {"exported": True, "n_features": top_n, "path": str(shap_dir)} + logger.info( + "SHAP export: top feature={} (|SHAP|={:.4f}), saved to {}", + shap_importance[0][0], + shap_importance[0][1], + shap_dir, + ) + except Exception as exc: + logger.warning("SHAP feature importance export skipped: {}", exc) + shap_artifact["error"] = str(exc) + return shap_artifact + + +def _base_replay_report(reason: str = "not_run") -> dict[str, Any]: + return { "enabled": False, - "reason": "not_run", + "reason": reason, "rows": [], "selected_trial": None, "selected_params": None, } - if not bool(hpo_cfg.get("enabled", True)): - report["reason"] = "hpo_disabled" - return report + +def _load_optuna_study_for_replay(hpo_cfg: dict[str, Any]) -> tuple[Any | None, str | None]: storage = hpo_cfg.get("study_storage") study_name = resolve_optuna_study_name(hpo_cfg.get("study_name")) if not storage or not study_name: - report["reason"] = "missing_study_storage_or_name" - return report - + return None, "missing_study_storage_or_name" try: import optuna except Exception: - report["reason"] = "optuna_unavailable" - return report - + return None, "optuna_unavailable" try: - study = optuna.load_study(study_name=str(study_name), storage=str(storage)) + return optuna.load_study(study_name=str(study_name), storage=str(storage)), None except Exception as exc: - report["reason"] = f"study_load_failed: {exc}" - return report + return None, f"study_load_failed: {exc}" - complete = [t for t in study.trials if t.state.name == "COMPLETE" and t.value is not None] - if not complete: - report["reason"] = "no_complete_trials" - return report - top = sorted( +def _top_complete_trials(study: Any, top_k_trials: int) -> list[Any]: + complete = [t for t in study.trials if t.state.name == "COMPLETE" and t.value is not None] + return sorted( complete, key=lambda t: float(t.value if t.value is not None else float("-inf")), reverse=True, )[: max(1, int(top_k_trials))] + + +def _trial_gate_attrs(trial: Any) -> tuple[Any, Any, Any, bool, bool | None]: + fairness_pass_attr = trial.user_attrs.get("fairness_pass") + conformal_pass_attr = trial.user_attrs.get("conformal_pass") + governance_pass_attr = trial.user_attrs.get("governance_pass") + gate_attrs_present = all( + key in trial.user_attrs for key in ("fairness_pass", "conformal_pass", "governance_pass") + ) + gate_all_pass = ( + bool(fairness_pass_attr and conformal_pass_attr and governance_pass_attr) + if gate_attrs_present + else None + ) + return ( + fairness_pass_attr, + conformal_pass_attr, + governance_pass_attr, + bool(gate_attrs_present), + gate_all_pass, + ) + + +def _replay_trial_seed_row( + *, + trial: Any, + seed: int, + base_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + cat_features: list[str], + sample_weight: np.ndarray | None, + eval_sample_weight: np.ndarray | None, +) -> dict[str, Any]: + ( + fairness_pass_attr, + conformal_pass_attr, + governance_pass_attr, + gate_attrs_present, + gate_all_pass, + ) = _trial_gate_attrs(trial) + params = {**base_params, **trial.params, "random_seed": int(seed)} + model, metrics = train_catboost_default( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + cat_features=cat_features, + params=params, + sample_weight=sample_weight, + eval_sample_weight=eval_sample_weight, + ) + y_val_prob = model.predict_proba(X_val_cb)[:, 1] + y_val_array = y_val.to_numpy(dtype=int) + return { + "trial_number": int(trial.number), + "seed": int(seed), + "validation_auc": _metric_float(metrics, "validation_auc"), + "validation_brier": float(brier_score_loss(y_val_array, y_val_prob)), + "validation_ece": float(expected_calibration_error(y_val_array, y_val_prob)), + "best_iteration": _metric_int(metrics, "best_iteration"), + "trial_best_value": float(trial.value if trial.value is not None else float("nan")), + "fairness_pass": fairness_pass_attr, + "conformal_pass": conformal_pass_attr, + "governance_pass": governance_pass_attr, + "gate_attrs_present": gate_attrs_present, + "gate_all_pass": gate_all_pass, + "params": trial.params, + } + + +def _replay_rows_for_trials( + *, + top_trials: list[Any], + seeds: list[int], + base_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + cat_features: list[str], + sample_weight: np.ndarray | None, + eval_sample_weight: np.ndarray | None, +) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] - for trial in top: - fairness_pass_attr = trial.user_attrs.get("fairness_pass") - conformal_pass_attr = trial.user_attrs.get("conformal_pass") - governance_pass_attr = trial.user_attrs.get("governance_pass") - gate_attrs_present = all( - key in trial.user_attrs - for key in ("fairness_pass", "conformal_pass", "governance_pass") - ) - gate_all_pass = ( - bool(fairness_pass_attr and conformal_pass_attr and governance_pass_attr) - if gate_attrs_present - else None - ) + for trial in top_trials: for seed in seeds: - params = {**base_params, **trial.params, "random_seed": int(seed)} - model, metrics = train_catboost_default( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - cat_features=cat_features, - params=params, - sample_weight=sample_weight, - eval_sample_weight=eval_sample_weight, - ) - y_val_prob = model.predict_proba(X_val_cb)[:, 1] rows.append( - { - "trial_number": int(trial.number), - "seed": int(seed), - "validation_auc": float(metrics.get("validation_auc", 0.0)), - "validation_brier": float(brier_score_loss(y_val, y_val_prob)), - "validation_ece": float(expected_calibration_error(y_val, y_val_prob)), - "best_iteration": int(metrics.get("best_iteration", 0)), - "trial_best_value": float( - trial.value if trial.value is not None else float("nan") - ), - "fairness_pass": fairness_pass_attr, - "conformal_pass": conformal_pass_attr, - "governance_pass": governance_pass_attr, - "gate_attrs_present": bool(gate_attrs_present), - "gate_all_pass": gate_all_pass, - "params": trial.params, - } + _replay_trial_seed_row( + trial=trial, + seed=seed, + base_params=base_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + cat_features=cat_features, + sample_weight=sample_weight, + eval_sample_weight=eval_sample_weight, + ) ) + return rows + + +def _gate_tier(gate_all_pass_summary: bool | None, prioritize_gate_pass: bool) -> int: + if not prioritize_gate_pass: + return 1 + if gate_all_pass_summary is True: + return 0 + if gate_all_pass_summary is None: + return 1 + return 2 + + +def _trial_summary_row( + trial_number: Any, + grp: pd.DataFrame, + *, + prioritize_gate_pass: bool, +) -> dict[str, Any]: + gate_present = bool(grp["gate_attrs_present"].fillna(False).any()) + gate_values = grp["gate_all_pass"].dropna().astype(bool) + gate_all_pass_summary = None if gate_values.empty else bool(gate_values.all()) + return { + "trial_number": int(str(trial_number)), + "median_validation_auc": float(grp["validation_auc"].median()), + "mean_validation_auc": float(grp["validation_auc"].mean()), + "std_validation_auc": float(grp["validation_auc"].std(ddof=0)), + "mean_validation_brier": float(grp["validation_brier"].mean()), + "mean_validation_ece": float(grp["validation_ece"].mean()), + "gate_attrs_present": gate_present, + "gate_all_pass": gate_all_pass_summary, + "gate_tier": int(_gate_tier(gate_all_pass_summary, prioritize_gate_pass)), + } + + +def _summarize_replayed_trials( + rows: list[dict[str, Any]], + *, + prioritize_gate_pass: bool, +) -> pd.DataFrame: + replay_df = pd.DataFrame(rows) + summary_rows = [ + _trial_summary_row( + trial_number, + grp, + prioritize_gate_pass=prioritize_gate_pass, + ) + for trial_number, grp in replay_df.groupby("trial_number", observed=True) + ] + return pd.DataFrame(summary_rows).sort_values( + [ + "gate_tier", + "mean_validation_ece", + "median_validation_auc", + "mean_validation_brier", + ], + ascending=[True, True, False, True], + ) + + +def _replay_selection_policy(prioritize_gate_pass: bool) -> dict[str, Any]: + return { + "prioritize_gate_pass": bool(prioritize_gate_pass), + "rank_order": [ + "gate_tier(pass->unknown->fail)", + "mean_validation_ece(asc)", + "median_validation_auc(desc)", + "mean_validation_brier(asc)", + ], + } + +def _replay_top_optuna_trials( + *, + hpo_cfg: dict[str, Any], + base_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + cat_features: list[str], + seeds: list[int], + top_k_trials: int = 3, + prioritize_gate_pass: bool = True, + sample_weight: np.ndarray | None = None, + eval_sample_weight: np.ndarray | None = None, +) -> dict[str, Any]: + """Replay top Optuna trials across multiple seeds for robustness.""" + report = _base_replay_report() + if not bool(hpo_cfg.get("enabled", True)): + report["reason"] = "hpo_disabled" + return report + + study, reason = _load_optuna_study_for_replay(hpo_cfg) + if study is None: + report["reason"] = reason or "study_load_failed" + return report + + top = _top_complete_trials(study, top_k_trials) + if not top: + report["reason"] = "no_complete_trials" + return report + + rows = _replay_rows_for_trials( + top_trials=top, + seeds=seeds, + base_params=base_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + cat_features=cat_features, + sample_weight=sample_weight, + eval_sample_weight=eval_sample_weight, + ) if not rows: report["reason"] = "no_replay_rows" return report - replay_df = pd.DataFrame(rows) + grouped = _summarize_replayed_trials( + rows, + prioritize_gate_pass=prioritize_gate_pass, + ) + selected_trial = int(grouped.iloc[0]["trial_number"]) + selected_trial_obj = next(t for t in top if int(t.number) == selected_trial) + selected_params = {**base_params, **selected_trial_obj.params} + + report.update( + { + "enabled": True, + "reason": "ok", + "top_k_trials": len(top), + "seeds": [int(s) for s in seeds], + "selection_policy": _replay_selection_policy(prioritize_gate_pass), + "rows": rows, + "summary_by_trial": grouped.to_dict(orient="records"), + "selected_trial": selected_trial, + "selected_params": selected_params, + } + ) + return report + + +def _initial_seed_replay_report(reason: str = "disabled_in_config") -> dict[str, Any]: + return { + "enabled": False, + "reason": reason, + "rows": [], + "selected_trial": None, + "selected_params": None, + } + + +def _train_replay_manifest_catboost( + *, + replay_cfg: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any], dict[str, Any]]: + replay_params = dict(replay_cfg.get("selected_params") or {}) + model, metrics = train_catboost_default( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + X_test=X_test_cb, + y_test=y_test, + cat_features=categorical_features, + params=replay_params, + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + ) + metrics["hpo_trials_executed"] = 0 + metrics["hpo_best_validation_auc"] = float(metrics["validation_auc"]) + metrics["best_params"] = replay_params + metrics["model_type"] = "catboost_replay_manifest" + seed_replay_report = _initial_seed_replay_report("replay_manifest") + seed_replay_report["selected_params"] = replay_params + return model, metrics, seed_replay_report + + +def _train_default_as_tuned_catboost( + *, + model_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any], dict[str, Any]]: + model, metrics = train_catboost_default( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + X_test=X_test_cb, + y_test=y_test, + cat_features=categorical_features, + params=model_params, + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + ) + metrics["hpo_trials_executed"] = 0 + metrics["hpo_best_validation_auc"] = float(metrics["validation_auc"]) + metrics["best_params"] = model_params + return model, metrics, _initial_seed_replay_report("hpo_disabled") + + +def _maybe_apply_seed_replay_selection( + *, + seed_replay_report: dict[str, Any], + current_metrics: dict[str, Any], + current_model: Any, + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any]]: + if not seed_replay_report.get("enabled") or not seed_replay_report.get("selected_params"): + return current_model, current_metrics + selected_params = dict(seed_replay_report["selected_params"]) + replay_model, replay_metrics = train_catboost_default( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + X_test=X_test_cb, + y_test=y_test, + cat_features=categorical_features, + params=selected_params, + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + ) + return replay_model, { + **current_metrics, + **replay_metrics, + "model_type": "catboost_tuned_seed_replay_selected", + "best_params": selected_params, + "seed_replay_selected_trial": seed_replay_report.get("selected_trial"), + "seed_replay_enabled": True, + } - summary_rows: list[dict[str, Any]] = [] - for trial_number, grp in replay_df.groupby("trial_number", observed=True): - gate_present = bool(grp["gate_attrs_present"].fillna(False).any()) - gate_values = grp["gate_all_pass"].dropna().astype(bool) - if gate_values.empty: - gate_all_pass_summary: bool | None = None - else: - gate_all_pass_summary = bool(gate_values.all()) - - if not prioritize_gate_pass: - gate_tier = 1 - elif gate_all_pass_summary is True: - gate_tier = 0 - elif gate_all_pass_summary is None: - gate_tier = 1 - else: - gate_tier = 2 - summary_rows.append( - { - "trial_number": int(trial_number), - "median_validation_auc": float(grp["validation_auc"].median()), - "mean_validation_auc": float(grp["validation_auc"].mean()), - "std_validation_auc": float(grp["validation_auc"].std(ddof=0)), - "mean_validation_brier": float(grp["validation_brier"].mean()), - "mean_validation_ece": float(grp["validation_ece"].mean()), - "gate_attrs_present": gate_present, - "gate_all_pass": gate_all_pass_summary, - "gate_tier": int(gate_tier), - } +def _train_hpo_catboost( + *, + hpo_cfg: dict[str, Any], + seed_replay_cfg: dict[str, Any], + model_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any], dict[str, Any]]: + model, metrics = train_catboost_tuned_optuna( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + X_test=X_test_cb, + y_test=y_test, + cat_features=categorical_features, + base_params=model_params, + n_trials=int(hpo_cfg.get("n_trials", 100)), + sampler=str(hpo_cfg.get("sampler", "tpe")), + pruner=str(hpo_cfg.get("pruner", "median")), + timeout_minutes=int(hpo_cfg.get("timeout_minutes", 0)), + n_startup_trials=int(hpo_cfg.get("n_startup_trials", 40)), + multivariate_tpe=bool(hpo_cfg.get("multivariate_tpe", True)), + group_tpe=bool(hpo_cfg.get("group_tpe", True)), + warn_independent_sampling=bool(hpo_cfg.get("warn_independent_sampling", True)), + constant_liar=bool(hpo_cfg.get("constant_liar", False)), + pruner_n_startup_trials=int(hpo_cfg.get("pruner_n_startup_trials", 20)), + pruner_n_warmup_steps=int(hpo_cfg.get("pruner_n_warmup_steps", 50)), + use_pruning_callback=bool(hpo_cfg.get("use_pruning_callback", True)), + study_storage=hpo_cfg.get("study_storage"), + study_name=hpo_cfg.get("study_name"), + load_if_exists=bool(hpo_cfg.get("load_if_exists", True)), + refit_full_train=bool(hpo_cfg.get("refit_full_train", True)), + gc_after_trial=bool(hpo_cfg.get("gc_after_trial", True)), + storage_heartbeat_interval=int(hpo_cfg.get("storage_heartbeat_interval", 0)), + storage_grace_period=int(hpo_cfg.get("storage_grace_period", 0)), + sqlite_timeout_seconds=int(hpo_cfg.get("sqlite_timeout_seconds", 60)), + retry_failed_trials=int(hpo_cfg.get("retry_failed_trials", 0)), + n_jobs=int(hpo_cfg.get("n_jobs", 1)), + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + search_space_mode=str(hpo_cfg.get("search_space_mode", "global")), + local_refine_space=dict(hpo_cfg.get("local_refine", {}) or {}), + constraints_policy=dict(hpo_cfg.get("constraints_policy", {}) or {}), + search_space_version=str(hpo_cfg.get("search_space_version", "cb_space_v2")), + ) + seed_replay_report = _initial_seed_replay_report() + if bool(seed_replay_cfg.get("enabled", True)): + seed_replay_report = _replay_top_optuna_trials( + hpo_cfg=hpo_cfg, + base_params=model_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + cat_features=categorical_features, + seeds=[int(seed) for seed in seed_replay_cfg.get("seeds", [42, 52, 62])], + top_k_trials=int(seed_replay_cfg.get("top_k_trials", 3)), + prioritize_gate_pass=bool(seed_replay_cfg.get("prioritize_gate_pass", True)), + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + ) + model, metrics = _maybe_apply_seed_replay_selection( + seed_replay_report=seed_replay_report, + current_metrics=metrics, + current_model=model, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, ) + return model, metrics, seed_replay_report - grouped = pd.DataFrame(summary_rows).sort_values( - [ - "gate_tier", - "mean_validation_ece", - "median_validation_auc", - "mean_validation_brier", - ], - ascending=[True, True, False, True], - ) - selected_trial = int(grouped.iloc[0]["trial_number"]) - selected_trial_obj = next(t for t in top if int(t.number) == selected_trial) - selected_params = {**base_params, **selected_trial_obj.params} - report.update( - { - "enabled": True, - "reason": "ok", - "top_k_trials": len(top), - "seeds": [int(s) for s in seeds], - "selection_policy": { - "prioritize_gate_pass": bool(prioritize_gate_pass), - "rank_order": [ - "gate_tier(pass->unknown->fail)", - "mean_validation_ece(asc)", - "median_validation_auc(desc)", - "mean_validation_brier(asc)", - ], - }, - "rows": rows, - "summary_by_trial": grouped.to_dict(orient="records"), - "selected_trial": selected_trial, - "selected_params": selected_params, - } +def _train_tuned_catboost_stage( + *, + run_mode: str, + replay_cfg: dict[str, Any], + hpo_cfg: dict[str, Any], + seed_replay_cfg: dict[str, Any], + model_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any], dict[str, Any]]: + if run_mode == "replay": + return _train_replay_manifest_catboost( + replay_cfg=replay_cfg, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, + ) + if bool(hpo_cfg.get("enabled", True)): + return _train_hpo_catboost( + hpo_cfg=hpo_cfg, + seed_replay_cfg=seed_replay_cfg, + model_params=model_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, + ) + return _train_default_as_tuned_catboost( + model_params=model_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, ) - return report def main( @@ -1560,138 +2398,29 @@ def main( # CatBoost tuned (Optuna) hpo_cfg = config.get("hpo", {}) - seed_replay_report: dict[str, Any] = { - "enabled": False, - "reason": "disabled_in_config", - "rows": [], - "selected_trial": None, - "selected_params": None, - } - if run_mode == "replay": - replay_params = dict(replay_cfg.get("selected_params") or {}) - cb_tuned_model, cb_tuned_metrics = train_catboost_default( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - X_test=X_test_cb, - y_test=y_test, - cat_features=categorical_features, - params=replay_params, - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - ) - cb_tuned_metrics["hpo_trials_executed"] = 0 - cb_tuned_metrics["hpo_best_validation_auc"] = float(cb_tuned_metrics["validation_auc"]) - cb_tuned_metrics["best_params"] = replay_params - cb_tuned_metrics["model_type"] = "catboost_replay_manifest" - seed_replay_report["reason"] = "replay_manifest" - seed_replay_report["selected_params"] = replay_params - elif bool(hpo_cfg.get("enabled", True)): - cb_tuned_model, cb_tuned_metrics = train_catboost_tuned_optuna( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - X_test=X_test_cb, - y_test=y_test, - cat_features=categorical_features, - base_params=config["model"].get("params", {}), - n_trials=int(hpo_cfg.get("n_trials", 100)), - sampler=str(hpo_cfg.get("sampler", "tpe")), - pruner=str(hpo_cfg.get("pruner", "median")), - timeout_minutes=int(hpo_cfg.get("timeout_minutes", 0)), - n_startup_trials=int(hpo_cfg.get("n_startup_trials", 40)), - multivariate_tpe=bool(hpo_cfg.get("multivariate_tpe", True)), - group_tpe=bool(hpo_cfg.get("group_tpe", True)), - warn_independent_sampling=bool(hpo_cfg.get("warn_independent_sampling", True)), - constant_liar=bool(hpo_cfg.get("constant_liar", False)), - pruner_n_startup_trials=int(hpo_cfg.get("pruner_n_startup_trials", 20)), - pruner_n_warmup_steps=int(hpo_cfg.get("pruner_n_warmup_steps", 50)), - use_pruning_callback=bool(hpo_cfg.get("use_pruning_callback", True)), - study_storage=hpo_cfg.get("study_storage", None), - study_name=hpo_cfg.get("study_name", None), - load_if_exists=bool(hpo_cfg.get("load_if_exists", True)), - refit_full_train=bool(hpo_cfg.get("refit_full_train", True)), - gc_after_trial=bool(hpo_cfg.get("gc_after_trial", True)), - storage_heartbeat_interval=int(hpo_cfg.get("storage_heartbeat_interval", 0)), - storage_grace_period=int(hpo_cfg.get("storage_grace_period", 0)), - sqlite_timeout_seconds=int(hpo_cfg.get("sqlite_timeout_seconds", 60)), - retry_failed_trials=int(hpo_cfg.get("retry_failed_trials", 0)), - n_jobs=int(hpo_cfg.get("n_jobs", 1)), - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - search_space_mode=str(hpo_cfg.get("search_space_mode", "global")), - local_refine_space=dict(hpo_cfg.get("local_refine", {}) or {}), - constraints_policy=dict(hpo_cfg.get("constraints_policy", {}) or {}), - search_space_version=str(hpo_cfg.get("search_space_version", "cb_space_v2")), - ) - - if bool(seed_replay_cfg.get("enabled", True)): - seeds = seed_replay_cfg.get("seeds", [42, 52, 62]) - seeds = [int(s) for s in seeds] - prioritize_gate_pass = bool(seed_replay_cfg.get("prioritize_gate_pass", True)) - seed_replay_report = _replay_top_optuna_trials( - hpo_cfg=hpo_cfg, - base_params=config["model"].get("params", {}), - X_train_fit_cb=X_train_fit_cb, - y_train_fit=y_train_fit, - X_val_cb=X_val_cb, - y_val=y_val, - cat_features=categorical_features, - seeds=seeds, - top_k_trials=int(seed_replay_cfg.get("top_k_trials", 3)), - prioritize_gate_pass=prioritize_gate_pass, - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - ) - if seed_replay_report.get("enabled") and seed_replay_report.get("selected_params"): - selected_params = dict(seed_replay_report["selected_params"]) - replay_model, replay_metrics = train_catboost_default( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - X_test=X_test_cb, - y_test=y_test, - cat_features=categorical_features, - params=selected_params, - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - ) - cb_tuned_metrics = { - **cb_tuned_metrics, - **replay_metrics, - "model_type": "catboost_tuned_seed_replay_selected", - "best_params": selected_params, - "seed_replay_selected_trial": seed_replay_report.get("selected_trial"), - "seed_replay_enabled": True, - } - cb_tuned_model = replay_model - else: - cb_tuned_model, cb_tuned_metrics = train_catboost_default( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - X_test=X_test_cb, - y_test=y_test, - cat_features=categorical_features, - params=config["model"].get("params", {}), - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - ) - cb_tuned_metrics["hpo_trials_executed"] = 0 - cb_tuned_metrics["hpo_best_validation_auc"] = float(cb_tuned_metrics["validation_auc"]) - cb_tuned_metrics["best_params"] = config["model"].get("params", {}) - seed_replay_report["reason"] = "hpo_disabled" + cb_tuned_model, cb_tuned_metrics, seed_replay_report = _train_tuned_catboost_stage( + run_mode=run_mode, + replay_cfg=replay_cfg, + hpo_cfg=hpo_cfg, + seed_replay_cfg=seed_replay_cfg, + model_params=dict(config["model"].get("params", {})), + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, + ) _write_checkpoint( checkpoint_dir, "hpo_summary", { - "best_params": cb_tuned_metrics.get("best_params", {}), - "hpo_trials_executed": int(cb_tuned_metrics.get("hpo_trials_executed", 0)), - "hpo_best_validation_auc": float(cb_tuned_metrics.get("hpo_best_validation_auc", 0.0)), + "best_params": _metric_mapping(cb_tuned_metrics, "best_params", {}), + "hpo_trials_executed": _metric_int(cb_tuned_metrics, "hpo_trials_executed"), + "hpo_best_validation_auc": _metric_float(cb_tuned_metrics, "hpo_best_validation_auc"), "seed_replay_report": seed_replay_report, }, ) @@ -1701,35 +2430,23 @@ def main( state="running", config_path=config_path, extra={ - "validation_auc": float(cb_tuned_metrics.get("hpo_best_validation_auc", 0.0)), - "best_iteration": int(cb_tuned_metrics.get("best_iteration", 0)), + "validation_auc": _metric_float(cb_tuned_metrics, "hpo_best_validation_auc"), + "best_iteration": _metric_int(cb_tuned_metrics, "best_iteration"), }, ) - walk_forward_report: dict[str, Any] - if bool(walk_cfg.get("enabled", True)): - walk_forward_report = _evaluate_walk_forward_auc( - train, - features=catboost_features, - categorical_features=categorical_features, - target=TARGET, - params=dict(cb_tuned_metrics.get("best_params", config["model"].get("params", {}))), - n_windows=int(walk_cfg.get("n_windows", 3)), - min_train_rows=int(walk_cfg.get("min_train_rows", 200_000)), - window_rows=int(walk_cfg.get("window_rows", 80_000)), - date_col=str(walk_cfg.get("date_col", "issue_d")), - max_rows=( - None if int(walk_cfg.get("max_rows", 0)) <= 0 else int(walk_cfg.get("max_rows", 0)) - ), - ) - else: - walk_forward_report = { - "enabled": False, - "reason": "disabled_in_config", - "n_windows_requested": int(walk_cfg.get("n_windows", 0)), - "n_windows_used": 0, - "folds": [], - } + walk_forward_report = _evaluate_walk_forward_stage( + enabled=bool(walk_cfg.get("enabled", True)), + walk_cfg=walk_cfg, + train=train, + catboost_features=catboost_features, + categorical_features=categorical_features, + model_params=_metric_mapping( + cb_tuned_metrics, + "best_params", + dict(config["model"].get("params", {})), + ), + ) # Raw probabilities y_prob_default_test = cb_default_model.predict_proba(X_test_cb)[:, 1] @@ -1781,75 +2498,20 @@ def main( decision_cfg = config.get("decision_threshold", {}) resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - decision_threshold_artifact = { - "enabled": False, - "selected_threshold": float(config.get("calibration", {}).get("default_threshold", 0.5)), - "source": "fallback_default", - **build_artifact_metadata( - schema_version="2026-03-01.1", - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - if run_mode == "replay" and replay_cfg.get("decision_threshold_artifact"): - decision_threshold_artifact = dict(replay_cfg["decision_threshold_artifact"]) - decision_threshold_artifact.update( - build_artifact_metadata( - schema_version="2026-03-01.1", - run_tag=resolved_run_tag, - require_explicit=True, - ) - ) - decision_threshold_artifact["source"] = "frozen_replay_manifest" - elif bool(decision_cfg.get("enabled", True)): - fairness_policy_path = str( - decision_cfg.get("fairness_policy_path", "configs/fairness_policy.yaml") - ) - with open(fairness_policy_path, encoding="utf-8") as f: - fairness_cfg = yaml.safe_load(f) or {} - fairness_policy = fairness_cfg.get("policy", {}) - fairness_attrs = fairness_cfg.get("attributes", []) - groups_for_threshold = _build_fairness_groups_for_threshold(train_val, fairness_attrs) - groups_for_threshold_cal = _build_fairness_groups_for_threshold(cal, fairness_attrs) - - thr_min = float(decision_cfg.get("min_threshold", 0.05)) - thr_max = float(decision_cfg.get("max_threshold", 0.95)) - thr_step = float(decision_cfg.get("step", 0.01)) - thresholds = np.arange(thr_min, thr_max + (thr_step / 2.0), thr_step) - fallback_threshold = float(fairness_policy.get("prediction_threshold", 0.5)) - threshold_result = _select_decision_threshold( - y_true=y_val.to_numpy(), - y_prob=y_prob_final_val, - policy={ - "dpd_threshold": float(fairness_policy.get("dpd_threshold", 0.10)), - "eo_gap_threshold": float(fairness_policy.get("eo_gap_threshold", 0.10)), - "dir_threshold": float(fairness_policy.get("dir_threshold", 0.80)), - }, - groups_dict=groups_for_threshold, - thresholds=np.asarray(thresholds, dtype=float), - fallback_threshold=fallback_threshold, - y_true_secondary=y_cal.to_numpy(), - y_prob_secondary=y_prob_tuned_cal, - groups_dict_secondary=groups_for_threshold_cal, - ) - - decision_threshold_artifact = { - "enabled": True, - "selected_threshold": float(threshold_result["selected_threshold"]), - "fallback_threshold": fallback_threshold, - "selection_metrics": threshold_result["selection_metrics"], - "search_summary": threshold_result["search_summary"], - "source": "validation_fairness_search", - "fairness_policy_path": fairness_policy_path, - "validation_rows": len(train_val), - "secondary_validation_rows": len(cal), - "calibration_method": selected_cal_method, - **build_artifact_metadata( - schema_version="2026-03-01.1", - run_tag=resolved_run_tag, - require_explicit=True, - ), - } + decision_threshold_artifact = _resolve_decision_threshold_artifact( + config=config, + run_mode=run_mode, + replay_cfg=replay_cfg, + decision_cfg=decision_cfg, + train_val=train_val, + cal=cal, + y_val=y_val, + y_prob_final_val=y_prob_final_val, + y_cal=y_cal, + y_prob_tuned_cal=y_prob_tuned_cal, + selected_cal_method=selected_cal_method, + run_tag=resolved_run_tag, + ) # Conformal (keeps calibration split isolated from model training) alpha = 1.0 - float(config["conformal"].get("confidence_level", 0.9)) @@ -1870,83 +2532,11 @@ def main( # Statistical calibration hypothesis tests (MAPIE) # Tests H0: scores are well-calibrated. High p-value → well calibrated. - statistical_cal_tests: dict[str, object] = {} - try: - from mapie.metrics.calibration import ( - cumulative_differences, - kolmogorov_smirnov_p_value, - kuiper_p_value, - length_scale, - spiegelhalter_p_value, - ) - - y_test_arr = y_test.values.astype(float) - # Calibrated (champion) vs uncalibrated - for tag, probs in [("calibrated", y_prob_final), ("uncalibrated", y_prob_tuned_test)]: - try: - ks_p = float(kolmogorov_smirnov_p_value(y_test_arr, probs)) - ku_p = float(kuiper_p_value(y_test_arr, probs)) - sp_p = float(spiegelhalter_p_value(y_test_arr, probs)) - cum_diff = cumulative_differences(y_test_arr, probs) - sigma = float(length_scale(probs)) - statistical_cal_tests[tag] = { - "ks_pvalue": ks_p, - "kuiper_pvalue": ku_p, - "spiegelhalter_pvalue": sp_p, - "length_scale_sigma": sigma, - "n": len(y_test_arr), - } - logger.info( - f"Calibration tests [{tag}]: KS_p={ks_p:.4f} " - f"Kuiper_p={ku_p:.4f} Spiegelhalter_p={sp_p:.4f}" - ) - # Save cumulative differences for the figure (calibrated only) - if tag == "calibrated": - statistical_cal_tests["_cum_diff_calibrated"] = cum_diff.tolist() - statistical_cal_tests["_sigma"] = sigma - elif tag == "uncalibrated": - statistical_cal_tests["_cum_diff_uncalibrated"] = cum_diff.tolist() - except Exception as exc_inner: - logger.warning(f"Statistical calibration tests [{tag}] failed: {exc_inner}") - statistical_cal_tests[tag] = {"error": str(exc_inner)} - - # Save cumulative differences parquet for figure generation - if "_cum_diff_calibrated" in statistical_cal_tests: - k_idx = np.arange(len(y_test_arr)) / len(y_test_arr) - sigma_raw = statistical_cal_tests.get("_sigma", 0.0) - sigma_val = ( - float(sigma_raw) if isinstance(sigma_raw, str | int | float | np.number) else 0.0 - ) - cum_diff_df = pd.DataFrame( - { - "k": k_idx, - "cum_diff_calibrated": statistical_cal_tests.pop("_cum_diff_calibrated"), - "cum_diff_uncalibrated": statistical_cal_tests.pop( - "_cum_diff_uncalibrated", - [float("nan")] * len(k_idx), - ), - "sigma_upper": sigma_val * 2, - "sigma_lower": -sigma_val * 2, - } - ) - cum_diff_path = _artifact_path("data/processed/calibration_cumulative_diffs.parquet") - cum_diff_path.parent.mkdir(parents=True, exist_ok=True) - cum_diff_df.to_parquet(cum_diff_path, index=False) - logger.info(f"Saved calibration cumulative diffs: {cum_diff_path}") - statistical_cal_tests.pop("_sigma", None) - - stat_cal_path = _artifact_path("data/processed/statistical_calibration_tests.json") - stat_cal_path.parent.mkdir(parents=True, exist_ok=True) - stat_cal_path.write_text( - json.dumps(statistical_cal_tests, indent=2, default=str), encoding="utf-8" - ) - logger.info(f"Saved statistical calibration tests: {stat_cal_path}") - except ImportError: - logger.warning( - "mapie.metrics.calibration not available — statistical calibration tests skipped." - ) - except Exception as exc: - logger.warning(f"Statistical calibration tests block failed: {exc}") + _run_statistical_calibration_tests( + y_test=y_test, + y_prob_final=y_prob_final, + y_prob_tuned_test=y_prob_tuned_test, + ) brier_decomposition_path = _artifact_path( config["output"].get( @@ -2095,55 +2685,13 @@ def main( save_contract(contract_payload, contract_path) # ── SHAP feature importance export (CatBoost native) ── - shap_artifact: dict[str, Any] = {"exported": False} - try: - from catboost import Pool as _SHAPPool - - shap_pool = _SHAPPool(X_test_cb, cat_features=categorical_features) - shap_raw = cb_tuned_model.get_feature_importance(type="ShapValues", data=shap_pool) - # ShapValues returns (n_samples, n_features + 1); last col = expected value - shap_values = shap_raw[:, :-1] - shap_expected = float(shap_raw[0, -1]) - - mean_abs_shap = np.abs(shap_values).mean(axis=0) - shap_importance = sorted( - zip(catboost_features, mean_abs_shap.tolist(), strict=False), - key=lambda x: x[1], - reverse=True, - ) - shap_dir = _artifact_path(config["output"].get("shap_dir", "reports/figures/shap")) - shap_dir.mkdir(parents=True, exist_ok=True) - - # Save raw SHAP values (compressed) - np.savez_compressed( - str(shap_dir / "shap_values_test.npz"), - shap_values=shap_values, - expected_value=np.array([shap_expected]), - feature_names=np.array(catboost_features), - ) - - # Save top-N importance as JSON for Streamlit/governance - top_n = min(20, len(shap_importance)) - shap_summary = { - "expected_value": shap_expected, - "n_samples": int(shap_values.shape[0]), - "n_features": int(shap_values.shape[1]), - "top_features": [ - {"feature": f, "mean_abs_shap": round(v, 6)} for f, v in shap_importance[:top_n] - ], - } - shap_summary_path = shap_dir / "shap_feature_importance.json" - shap_summary_path.write_text(json.dumps(shap_summary, indent=2), encoding="utf-8") - shap_artifact = {"exported": True, "n_features": top_n, "path": str(shap_dir)} - logger.info( - "SHAP export: top feature={} (|SHAP|={:.4f}), saved to {}", - shap_importance[0][0], - shap_importance[0][1], - shap_dir, - ) - except Exception as exc: - logger.warning("SHAP feature importance export skipped: {}", exc) - shap_artifact["error"] = str(exc) + shap_artifact = _export_shap_feature_importance( + cb_tuned_model=cb_tuned_model, + X_test_cb=X_test_cb, + categorical_features=categorical_features, + catboost_features=catboost_features, + shap_dir=_artifact_path(config["output"].get("shap_dir", "reports/figures/shap")), + ) # Persist test predictions for downstream contracts. test_predictions_path = _artifact_path( @@ -2189,10 +2737,10 @@ def main( strict=False, ) }, - "optuna_best_auc": float(cb_tuned_metrics.get("auc_roc", 0.0)), - "optuna_best_params": cb_tuned_metrics.get("best_params", {}), - "hpo_trials_executed": int(cb_tuned_metrics.get("hpo_trials_executed", 0)), - "hpo_best_validation_auc": float(cb_tuned_metrics.get("hpo_best_validation_auc", 0.0)), + "optuna_best_auc": _metric_float(cb_tuned_metrics, "auc_roc"), + "optuna_best_params": _metric_mapping(cb_tuned_metrics, "best_params", {}), + "hpo_trials_executed": _metric_int(cb_tuned_metrics, "hpo_trials_executed"), + "hpo_best_validation_auc": _metric_float(cb_tuned_metrics, "hpo_best_validation_auc"), "walk_forward_report": walk_forward_report, "seed_replay_report": seed_replay_report, "decision_threshold": decision_threshold_artifact, @@ -2221,10 +2769,10 @@ def main( ) seed_replay_status = { "selected_calibration_method": selected_cal_method, - "validation_auc": float(cb_tuned_metrics.get("hpo_best_validation_auc", 0.0)), - "oot_auc": float(final_test_metrics.get("auc_roc", 0.0)), - "brier": float(final_test_metrics.get("brier_score", 0.0)), - "ece": float(final_test_metrics.get("ece", 0.0)), + "validation_auc": _metric_float(cb_tuned_metrics, "hpo_best_validation_auc"), + "oot_auc": _metric_float(final_test_metrics, "auc_roc"), + "brier": _metric_float(final_test_metrics, "brier_score"), + "ece": _metric_float(final_test_metrics, "ece"), "replay": seed_replay_report, **build_artifact_metadata( schema_version="2026-03-13.1", diff --git a/scripts/validate_alpha_gamma_bound.py b/scripts/validate_alpha_gamma_bound.py index c1ddb76..4f93269 100644 --- a/scripts/validate_alpha_gamma_bound.py +++ b/scripts/validate_alpha_gamma_bound.py @@ -23,9 +23,14 @@ _parse_percent_series, ) from src.models.conformal_artifacts import load_conformal_intervals # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + compute_funded_certificate_metrics, +) +from src.optimization.policy import policy_segment_labels # noqa: E402 from src.optimization.portfolio_model import ( # noqa: E402 compute_effective_pd, optimize_portfolio_allocation, + solution_allocation_vector, ) DEFAULT_ALPHAS = [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] @@ -116,30 +121,6 @@ def _load_aligned_dataset( return aligned -def _policy_segment_labels(loans: pd.DataFrame, policy_mode: str) -> np.ndarray | None: - if str(policy_mode).strip().lower() not in { - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - }: - return None - grade = ( - loans["grade"].fillna("unknown").astype(str) - if "grade" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - term = ( - loans["term"].fillna("unknown").astype(str) - if "term" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - verification = ( - loans["verification_status"].fillna("unknown").astype(str) - if "verification_status" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - return (grade + "|" + term + "|" + verification).to_numpy(dtype=object) - - def _compute_intervals_at_alpha( frame: pd.DataFrame, alpha: float, @@ -179,7 +160,7 @@ def _compute_effective_pd_vector( gamma=float(policy["gamma"]), delta_cap_quantile=float(policy["delta_cap_quantile"]), tail_focus_quantile=float(policy["tail_focus_quantile"]), - segment_labels=_policy_segment_labels(loans, str(policy["policy_mode"])), + segment_labels=policy_segment_labels(loans, str(policy["policy_mode"])), ) @@ -277,12 +258,7 @@ def _compute_exact_weights( if "loan_amnt" in loans.columns else np.ones(len(loans), dtype=float) ) - if "allocation_vector" in solution: - alloc = np.asarray(solution["allocation_vector"], dtype=float) - else: - alloc = np.array( - [float(solution["allocation"].get(i, 0.0)) for i in range(len(loans))], dtype=float - ) + alloc = solution_allocation_vector(solution, len(loans)) total_allocated = float(np.sum(alloc * loan_amounts)) weights = (alloc * loan_amounts) / max(total_allocated, 1e-6) return weights, { @@ -339,37 +315,52 @@ def _validate_single_alpha( else: raise ValueError(f"Unsupported allocator-mode={allocator_mode!r}") - miscoverage = (y_true > pd_high).astype(float) - V = float(np.sum(weights * miscoverage)) - weighted_pd_true = float(np.sum(weights * y_true)) - violation = max(0.0, weighted_pd_true - float(policy["risk_tolerance"])) - funded_mask = weights > 1e-8 - emp_coverage = ( - float(1.0 - miscoverage[funded_mask].mean()) if funded_mask.any() else float("nan") + certificate = compute_funded_certificate_metrics( + weights, + outcomes=y_true, + pd_point=pd_point, + pd_high=pd_high, + pd_effective=effective_pd, + alpha=alpha, + risk_tolerance=float(policy["risk_tolerance"]), + pd_cap_slack=float(alloc_meta.get("pd_cap_slack", 0.0)), ) - sqrt_alpha = float(np.sqrt(alpha)) bound_b_value = min(1.0, alpha / max(t_eval, 1e-8)) + risk_excess = round(certificate.realized_risk_tolerance_excess, 6) + empirical_risk_screen = bool(certificate.realized_risk_tolerance_excess <= alpha + 1e-8) + markov_screen = bool(certificate.sqrt_alpha + 1e-8 >= certificate.weighted_miscoverage) return { "alpha": float(alpha), "confidence": float(1.0 - alpha), - "gamma_cp": round(float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))), 6), - "n_funded": int(np.sum(funded_mask)), - "weighted_pd_true": round(weighted_pd_true, 6), - "weighted_pd_constraint_used": round(float(alloc_meta["weighted_pd_constraint_used"]), 6), - "weighted_pd_high": round(float(alloc_meta["weighted_pd_high"]), 6), - "weighted_pd_point": round(float(alloc_meta["weighted_pd_point"]), 6), + "gamma_cp": round(certificate.gamma_cp, 6), + "gamma_internalized": round(certificate.gamma_internalized, 6), + "gamma_residual": round(certificate.gamma_residual, 6), + "n_funded": certificate.n_funded, + "weighted_pd_true": round(certificate.weighted_outcome, 6), + "weighted_pd_constraint_used": round(certificate.weighted_pd_effective, 6), + "weighted_pd_high": round(certificate.endpoint_budget, 6), + "weighted_pd_point": round(certificate.weighted_pd_point, 6), + "endpoint_budget": round(certificate.endpoint_budget, 9), + "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), + "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), + "markov_loss_cap": round(certificate.markov_loss_cap, 9), "tau": float(policy["risk_tolerance"]), - "violation": round(violation, 6), - "weighted_miscoverage_V": round(V, 6), - "sqrt_alpha": round(sqrt_alpha, 6), - "empirical_coverage_funded": round(emp_coverage, 4), - "bound_a_expected_violation_leq_alpha": bool(violation <= alpha + 1e-8), + "realized_risk_tolerance_excess": risk_excess, + "violation": risk_excess, + "weighted_miscoverage_V": round(certificate.weighted_miscoverage, 6), + "weighted_coverage_funded": round(certificate.weighted_coverage, 6), + "sqrt_alpha": round(certificate.sqrt_alpha, 6), + "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 4), + "empirical_risk_excess_leq_alpha": empirical_risk_screen, + "bound_a_expected_violation_leq_alpha": empirical_risk_screen, "bound_b_prob_violation_gt_t": round(bound_b_value, 4), "bound_b_t_eval": float(t_eval), "bound_b_is_vacuous": bool(bound_b_value >= 1.0), - "bound_c_V_leq_sqrt_alpha": bool(sqrt_alpha + 1e-8 >= V), - "all_bounds_hold": bool((violation <= alpha + 1e-8) and (sqrt_alpha + 1e-8 >= V)), + "markov_miscoverage_screen_pass": markov_screen, + "bound_c_V_leq_sqrt_alpha": markov_screen, + "certificate_screen_pass": empirical_risk_screen and markov_screen, + "all_bounds_hold": empirical_risk_screen and markov_screen, "allocator_mode": mode, "solver_status": str(alloc_meta.get("solver_status", "unknown")), "allocator_solver_backend": str(alloc_meta.get("solver_backend", policy["solver_backend"])), @@ -404,22 +395,22 @@ def _plot_validation(results: list[dict[str, Any]], figure_prefix: Path) -> None ax = axes[1] ax.bar( df["alpha"], - df["violation"], + df["realized_risk_tolerance_excess"], width=0.012, color="#27ae60", alpha=0.7, - label="Violación empírica", + label="Realized risk-tolerance excess", ) ax.plot( df["alpha"], df["alpha"], "r--", linewidth=2, - label=r"Cota teórica $\mathbb{E}[\mathrm{violación}] \leq \alpha$", + label=r"Declared empirical screen: excess $\leq \alpha$", ) ax.set_xlabel(r"$\alpha$") - ax.set_ylabel("Violación de restricción PD") - ax.set_title(r"(B) Teorema 1(a): $\mathbb{E}[\mathrm{viol.}] \leq \alpha$") + ax.set_ylabel("Excess above risk tolerance") + ax.set_title("(B) Realized risk-tolerance screen") ax.legend(fontsize=8, loc="upper left") ax.grid(True, alpha=0.3) @@ -451,10 +442,10 @@ def _plot_validation(results: list[dict[str, Any]], figure_prefix: Path) -> None def _print_summary(results: list[dict[str, Any]], allocator_mode: str) -> None: all_pass = all(bool(r["all_bounds_hold"]) for r in results) print("\n" + "=" * 96) - print(f"VALIDATION SUMMARY: Theorem 1 ({allocator_mode.upper()} allocator)") + print(f"CERTIFICATE SUMMARY ({allocator_mode.upper()} allocator)") print("=" * 96) header = ( - f"{'α':>6} {'1-α':>6} {'Γ_CP':>8} {'Violation':>10} {'V':>8} " + f"{'α':>6} {'1-α':>6} {'Γ_CP':>8} {'Risk excess':>12} {'V':>8} " f"{'√α':>8} {'Mode':>8} {'Pass':>6}" ) print(header) @@ -463,11 +454,11 @@ def _print_summary(results: list[dict[str, Any]], allocator_mode: str) -> None: status = " ✓" if r["all_bounds_hold"] else " ✗" print( f"{r['alpha']:6.2f} {r['confidence']:6.2f} {r['gamma_cp']:8.4f} " - f"{r['violation']:10.6f} {r['weighted_miscoverage_V']:8.4f} " + f"{r['realized_risk_tolerance_excess']:10.6f} {r['weighted_miscoverage_V']:8.4f} " f"{r['sqrt_alpha']:8.4f} {r['allocator_mode']:>8} {status}" ) print("=" * 96) - print(f"Result: {'ALL BOUNDS HOLD' if all_pass else 'SOME BOUNDS FAILED'}") + print(f"Result: {'ALL SCREENS PASS' if all_pass else 'SOME SCREENS FAILED'}") def main(argv: list[str] | None = None) -> None: @@ -509,10 +500,10 @@ def main(argv: list[str] | None = None) -> None: results.append(result) status = "✓" if result["all_bounds_hold"] else "✗" logger.info( - " α={:.2f} Γ_CP={:.4f} violation={:.6f} V={:.4f} √α={:.4f} {}", + " α={:.2f} Γ_CP={:.4f} risk_excess={:.6f} V={:.4f} √α={:.4f} {}", alpha, result["gamma_cp"], - result["violation"], + result["realized_risk_tolerance_excess"], result["weighted_miscoverage_V"], result["sqrt_alpha"], status, @@ -524,7 +515,8 @@ def main(argv: list[str] | None = None) -> None: or (DEFAULT_EXACT_JSON if allocator_mode == "exact" else DEFAULT_PROXY_JSON) ) summary = { - "theorem": "Conformal Feasibility Guarantee (Theorem 1)", + "certificate": "Funded-set Markov accounting and empirical screens", + "theorem": "Distribution-free Markov bound under weighted funded-set validity", "paper": "CRPTO (CRPTO)", "allocator_mode": allocator_mode, "n_test_observations": len(aligned), @@ -532,6 +524,7 @@ def main(argv: list[str] | None = None) -> None: "conformal_intervals_path": str(args.conformal_intervals_path or ""), "alphas_tested": alpha_grid, "all_bounds_hold": all_pass, + "all_certificate_screens_pass": all_pass, "results": results, } output_json.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/validate_conformal_policy.py b/scripts/validate_conformal_policy.py index b22b862..31f1bd1 100644 --- a/scripts/validate_conformal_policy.py +++ b/scripts/validate_conformal_policy.py @@ -18,15 +18,17 @@ import yaml from loguru import logger +from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag +from src.utils.baseline_registry import resolve_official_baseline_run_tag + try: - from mapie.metrics.regression import regression_mwi_score as _mapie_mwi_score + from mapie.metrics.regression import regression_mwi_score as _imported_mapie_mwi_score + _mapie_mwi_score: Any = _imported_mapie_mwi_score _MAPIE_MWI_AVAILABLE = True except ImportError: _mapie_mwi_score = None _MAPIE_MWI_AVAILABLE = False -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag -from src.utils.baseline_registry import resolve_official_baseline_run_tag DEFAULT_POLICY_CONFIG = "configs/crpto_conformal_policy.yaml" @@ -50,9 +52,9 @@ def _fallback_winkler_interval_score( _imported_winkler_interval_score: Any try: - from src.evaluation.backtesting import ( - winkler_interval_score as _imported_winkler_interval_score, - ) + from src.evaluation import backtesting as _backtesting + + _imported_winkler_interval_score = _backtesting.winkler_interval_score except ImportError: _imported_winkler_interval_score = _fallback_winkler_interval_score @@ -140,233 +142,309 @@ def _apply_artifact_namespace( return updated -def main( - config_path: str = DEFAULT_POLICY_CONFIG, - run_tag: str | None = None, - sensitivity_config_path: str | None = None, - artifact_namespace: str | None = None, -) -> None: - with open(config_path, encoding="utf-8") as config_handle: - cfg_raw = yaml.safe_load(config_handle) or {} - cfg = dict(cast(Mapping[str, Any], cfg_raw)) +def _load_yaml_dict(path: str | Path) -> dict[str, Any]: + with open(path, encoding="utf-8") as handle: + raw = yaml.safe_load(handle) or {} + return dict(cast(Mapping[str, Any], raw)) + +def _load_policy_config( + *, + config_path: str, + sensitivity_config_path: str | None, + artifact_namespace: str | None, +) -> dict[str, Any]: + cfg = _load_yaml_dict(config_path) if sensitivity_config_path is not None: - with open(sensitivity_config_path, encoding="utf-8") as sensitivity_handle: - sens_raw = yaml.safe_load(sensitivity_handle) or {} - sens_cfg = dict(cast(Mapping[str, Any], sens_raw)) + sens_cfg = _load_yaml_dict(sensitivity_config_path) if "policy_sensitivity" in sens_cfg: cfg["policy_sensitivity"] = sens_cfg["policy_sensitivity"] logger.info( f"Overriding policy_sensitivity from {sensitivity_config_path}: " f"{cfg['policy_sensitivity']}" ) + return _apply_artifact_namespace(cfg, artifact_namespace) - cfg = _apply_artifact_namespace(cfg, artifact_namespace) - policy = dict(cast(Mapping[str, Any], cfg["policy"])) - artifacts = dict(cast(Mapping[str, Any], cfg["artifacts"])) - output = dict(cast(Mapping[str, Any], cfg["output"])) - resolved_run_tag = resolve_run_tag( - run_tag, - fallback_candidates=[resolve_official_baseline_run_tag()], - require_explicit=True, - ) +def _load_alerts(path: Path) -> pd.DataFrame: + if path.exists(): + return pd.read_parquet(path) + return pd.DataFrame({"severity": pd.Series(dtype="object")}) - with open(artifacts["conformal_results_path"], "rb") as results_handle: - results = cast(dict[str, Any], pickle.load(results_handle)) - group_metrics = pd.read_parquet(artifacts["group_metrics_path"]) - backtest_monthly = pd.read_parquet(artifacts["backtest_monthly_path"]) - alerts_path = Path(artifacts["backtest_alerts_path"]) - alerts = ( - pd.read_parquet(alerts_path) if alerts_path.exists() else pd.DataFrame(columns=["severity"]) - ) - intervals_path = Path( - artifacts.get("intervals_path", "data/processed/conformal_intervals_mondrian.parquet") - ) - intervals_df = pd.read_parquet(intervals_path) - lgd_ead_status_path = Path("models/conformal_lgd_ead_status.json") - lgd_ead_status = ( - json.loads(lgd_ead_status_path.read_text(encoding="utf-8")) - if lgd_ead_status_path.exists() - else {"available": False, "reason": "missing_status_artifact"} - ) - metrics_90 = results.get("metrics_90", {}) - metrics_95 = results.get("metrics_95", {}) +def _interval_arrays( + intervals_df: pd.DataFrame, + *, + lower_col: str, + upper_col: str, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + if {"y_true", lower_col, upper_col}.issubset(intervals_df.columns): + y_true = pd.to_numeric(intervals_df["y_true"], errors="coerce").to_numpy(dtype=float) + lower = pd.to_numeric(intervals_df[lower_col], errors="coerce").to_numpy(dtype=float) + upper = pd.to_numeric(intervals_df[upper_col], errors="coerce").to_numpy(dtype=float) + valid = np.isfinite(y_true) & np.isfinite(lower) & np.isfinite(upper) + return y_true[valid], lower[valid], upper[valid] + empty = np.array([], dtype=float) + return empty, empty, empty - coverage_90 = float(metrics_90.get("empirical_coverage", 0.0)) - coverage_95 = float(metrics_95.get("empirical_coverage", 0.0)) - avg_width_90 = float(metrics_90.get("avg_interval_width", 999.0)) - min_group_coverage_90 = float(group_metrics.get("coverage_90", pd.Series([0.0])).min()) - critical_alerts = int((alerts.get("severity", pd.Series([], dtype=str)) == "critical").sum()) - warning_alerts = int((alerts.get("severity", pd.Series([], dtype=str)) == "warning").sum()) - total_alerts = len(alerts) - # Conformal material-quality checks for the IJDS promotion contract. - if {"y_true", "pd_low_90", "pd_high_90"}.issubset(intervals_df.columns): - y_true = pd.to_numeric(intervals_df["y_true"], errors="coerce").to_numpy(dtype=float) - low_90 = pd.to_numeric(intervals_df["pd_low_90"], errors="coerce").to_numpy(dtype=float) - high_90 = pd.to_numeric(intervals_df["pd_high_90"], errors="coerce").to_numpy(dtype=float) - valid_90 = np.isfinite(y_true) & np.isfinite(low_90) & np.isfinite(high_90) - y90 = y_true[valid_90] - lo90 = low_90[valid_90] - hi90 = high_90[valid_90] - else: - y90 = np.array([], dtype=float) - lo90 = np.array([], dtype=float) - hi90 = np.array([], dtype=float) - - if {"y_true", "pd_low_95", "pd_high_95"}.issubset(intervals_df.columns): - y_true_95 = pd.to_numeric(intervals_df["y_true"], errors="coerce").to_numpy(dtype=float) - low_95 = pd.to_numeric(intervals_df["pd_low_95"], errors="coerce").to_numpy(dtype=float) - high_95 = pd.to_numeric(intervals_df["pd_high_95"], errors="coerce").to_numpy(dtype=float) - valid_95 = np.isfinite(y_true_95) & np.isfinite(low_95) & np.isfinite(high_95) - y95 = y_true_95[valid_95] - lo95 = low_95[valid_95] - hi95 = high_95[valid_95] - else: - y95 = np.array([], dtype=float) - lo95 = np.array([], dtype=float) - hi95 = np.array([], dtype=float) - - winkler_90 = ( - float(np.mean(winkler_interval_score(y90, lo90, hi90, alpha=0.10))) - if y90.size - else float("inf") - ) - winkler_95 = ( - float(np.mean(winkler_interval_score(y95, lo95, hi95, alpha=0.05))) - if y95.size - else float("inf") - ) +def _mean_winkler( + y_true: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + *, + alpha: float, +) -> float: + if not y_true.size: + return float("inf") + return float(np.mean(winkler_interval_score(y_true, lower, upper, alpha=alpha))) + + +def _violation_rate(y_true: np.ndarray, lower: np.ndarray, upper: np.ndarray) -> float: + if not y_true.size: + return float("nan") + return float(np.mean((y_true < lower) | (y_true > upper))) + - # Cross-check: MAPIE native regression_mwi_score (should match manual Winkler) - mapie_mwi_90: float | None = None - if _MAPIE_MWI_AVAILABLE and y90.size: - try: - mapie_mwi_90 = _compute_mapie_mwi_score( - y90, - lo90, - hi90, - confidence_level=0.90, +def _mapie_mwi_cross_check( + *, + y_true: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + winkler_score: float, + confidence_level: float, +) -> float | None: + if not (_MAPIE_MWI_AVAILABLE and y_true.size): + return None + try: + mapie_score = _compute_mapie_mwi_score( + y_true, + lower, + upper, + confidence_level=confidence_level, + ) + delta = abs(mapie_score - winkler_score) + if delta > 0.01: + logger.warning( + f"MAPIE MWI ({mapie_score:.4f}) deviates from manual Winkler " + f"({winkler_score:.4f}) by {delta:.4f} -- check score definition." ) - delta = abs(mapie_mwi_90 - winkler_90) - if delta > 0.01: - logger.warning( - f"MAPIE MWI ({mapie_mwi_90:.4f}) deviates from manual Winkler " - f"({winkler_90:.4f}) by {delta:.4f} — check score definition." - ) - else: - logger.info(f"MAPIE MWI cross-check OK: {mapie_mwi_90:.4f} ≈ {winkler_90:.4f}") - except Exception as exc: - logger.warning(f"MAPIE MWI cross-check failed: {exc}") - violation_rate_90 = float(np.mean((y90 < lo90) | (y90 > hi90))) if y90.size else float("nan") - violation_rate_95 = float(np.mean((y95 < lo95) | (y95 > hi95))) if y95.size else float("nan") + else: + logger.info(f"MAPIE MWI cross-check OK: {mapie_score:.4f} ~= {winkler_score:.4f}") + return mapie_score + except Exception as exc: + logger.warning(f"MAPIE MWI cross-check failed: {exc}") + return None + + +def _evaluate_check_frame(frame: pd.DataFrame) -> dict[str, Any]: + gate_pass = bool(frame["passed"].all()) + failing_material = frame.loc[~frame["passed"], "metric"].astype(str).tolist() + methodological_status = ( + "not_needed_material_gate_pass" if gate_pass else "blocked_material_gate_failures" + ) + return { + "strict_overall_pass": gate_pass, + "gate_overall_pass": gate_pass, + "non_statistical_checks_pass": gate_pass, + "diagnostic_statistical_pass": True, + "failing_checks": failing_material, + "failing_statistical_checks": [], + "gate_failing_checks": failing_material, + "failing_non_statistical_checks": failing_material, + "diagnostic_failing_checks": [], + "methodological_justification_pass": gate_pass, + "methodological_justification_status": methodological_status, + } + +def _winkler_90_check(policy: dict[str, Any], metrics: dict[str, float]) -> dict[str, object]: max_winkler_90 = float(policy.get("max_winkler_90", float("inf"))) - max_winkler_95 = float(policy.get("max_winkler_95", float("inf"))) - enable_compensated_winkler_90 = bool(policy.get("enable_compensated_winkler_90", False)) - compensated_winkler_90_max = float(policy.get("compensated_winkler_90_max", max_winkler_90)) - compensated_min_coverage_90 = float( + enable_compensated = bool(policy.get("enable_compensated_winkler_90", False)) + compensated_threshold = float(policy.get("compensated_winkler_90_max", max_winkler_90)) + compensated_min_coverage = float( policy.get("compensated_min_coverage_90", policy["target_coverage_90_min"]) ) - compensated_min_group_coverage_90 = float( - policy.get( - "compensated_min_group_coverage_90", - policy["min_group_coverage_90_min"], - ) + compensated_min_group_coverage = float( + policy.get("compensated_min_group_coverage_90", policy["min_group_coverage_90_min"]) ) - compensated_max_avg_width_90 = float( + compensated_max_avg_width = float( policy.get("compensated_max_avg_width_90", policy["max_avg_width_90"]) ) - winkler_90_raw_pass = bool(winkler_90 <= max_winkler_90) - winkler_90_compensated_pass = bool( - enable_compensated_winkler_90 - and (not winkler_90_raw_pass) - and winkler_90 <= compensated_winkler_90_max - and coverage_90 >= compensated_min_coverage_90 - and min_group_coverage_90 >= compensated_min_group_coverage_90 - and avg_width_90 <= compensated_max_avg_width_90 - and critical_alerts <= float(policy["max_critical_alerts"]) - ) - winkler_90_policy_pass = bool(winkler_90_raw_pass or winkler_90_compensated_pass) - winkler_90_policy_mode = ( - "strict" - if winkler_90_raw_pass - else "compensated_band" - if winkler_90_compensated_pass - else "strict" + raw_pass = bool(metrics["winkler_90"] <= max_winkler_90) + compensated_pass = bool( + enable_compensated + and (not raw_pass) + and metrics["winkler_90"] <= compensated_threshold + and metrics["coverage_90"] >= compensated_min_coverage + and metrics["min_group_coverage_90"] >= compensated_min_group_coverage + and metrics["avg_width_90"] <= compensated_max_avg_width + and metrics["critical_alerts"] <= float(policy["max_critical_alerts"]) ) - winkler_90_check = _check("winkler_90", winkler_90, max_winkler_90, "<=", "quality") - winkler_90_check["passed"] = bool(winkler_90_policy_pass) - winkler_90_check["policy_mode"] = str(winkler_90_policy_mode) - winkler_90_check["raw_threshold"] = float(max_winkler_90) - winkler_90_check["raw_passed"] = bool(winkler_90_raw_pass) - winkler_90_check["compensated_band_enabled"] = bool(enable_compensated_winkler_90) - winkler_90_check["compensated_threshold"] = float(compensated_winkler_90_max) - winkler_90_check["compensated_passed"] = bool(winkler_90_compensated_pass) - - checks = [ + policy_pass = bool(raw_pass or compensated_pass) + policy_mode = "strict" if raw_pass else "compensated_band" if compensated_pass else "strict" + check = _check("winkler_90", metrics["winkler_90"], max_winkler_90, "<=", "quality") + check["passed"] = bool(policy_pass) + check["policy_mode"] = str(policy_mode) + check["raw_threshold"] = float(max_winkler_90) + check["raw_passed"] = bool(raw_pass) + check["compensated_band_enabled"] = bool(enable_compensated) + check["compensated_threshold"] = float(compensated_threshold) + check["compensated_passed"] = bool(compensated_pass) + return check + + +def _policy_checks(policy: dict[str, Any], metrics: dict[str, float]) -> list[dict[str, object]]: + return [ _check( - "coverage_90", coverage_90, float(policy["target_coverage_90_min"]), ">=", "portfolio" + "coverage_90", + metrics["coverage_90"], + float(policy["target_coverage_90_min"]), + ">=", + "portfolio", ), _check( - "coverage_95", coverage_95, float(policy["target_coverage_95_min"]), ">=", "portfolio" + "coverage_95", + metrics["coverage_95"], + float(policy["target_coverage_95_min"]), + ">=", + "portfolio", ), _check( "min_group_coverage_90", - min_group_coverage_90, + metrics["min_group_coverage_90"], float(policy["min_group_coverage_90_min"]), ">=", "group", ), - _check("avg_width_90", avg_width_90, float(policy["max_avg_width_90"]), "<=", "portfolio"), + _check( + "avg_width_90", + metrics["avg_width_90"], + float(policy["max_avg_width_90"]), + "<=", + "portfolio", + ), _check( "critical_alerts", - float(critical_alerts), + metrics["critical_alerts"], float(policy["max_critical_alerts"]), "<=", "monitoring", ), _check( "total_alerts", - float(total_alerts), + metrics["total_alerts"], float(policy["max_total_alerts"]), "<=", "monitoring", ), _check( "warning_alerts", - float(warning_alerts), + metrics["warning_alerts"], float(policy["max_warning_alerts"]), "<=", "monitoring", ), - winkler_90_check, - _check("winkler_95", winkler_95, max_winkler_95, "<=", "quality"), + _winkler_90_check(policy, metrics), + _check( + "winkler_95", + metrics["winkler_95"], + float(policy.get("max_winkler_95", float("inf"))), + "<=", + "quality", + ), ] - checks_df = pd.DataFrame(checks) - def _evaluate_check_frame(frame: pd.DataFrame) -> dict[str, Any]: - gate_pass = bool(frame["passed"].all()) - failing_material = frame.loc[~frame["passed"], "metric"].astype(str).tolist() - methodological_status = ( - "not_needed_material_gate_pass" if gate_pass else "blocked_material_gate_failures" - ) - return { - "strict_overall_pass": gate_pass, - "gate_overall_pass": gate_pass, - "non_statistical_checks_pass": gate_pass, - "diagnostic_statistical_pass": True, - "failing_checks": failing_material, - "failing_statistical_checks": [], - "gate_failing_checks": failing_material, - "failing_non_statistical_checks": failing_material, - "diagnostic_failing_checks": [], - "methodological_justification_pass": gate_pass, - "methodological_justification_status": methodological_status, - } + +def _latest_backtest_month(backtest_monthly: pd.DataFrame) -> object | None: + if backtest_monthly.empty: + return None + return backtest_monthly.sort_values("month").iloc[-1]["month"] + + +def main( + config_path: str = DEFAULT_POLICY_CONFIG, + run_tag: str | None = None, + sensitivity_config_path: str | None = None, + artifact_namespace: str | None = None, +) -> None: + cfg = _load_policy_config( + config_path=config_path, + sensitivity_config_path=sensitivity_config_path, + artifact_namespace=artifact_namespace, + ) + + policy = dict(cast(Mapping[str, Any], cfg["policy"])) + artifacts = dict(cast(Mapping[str, Any], cfg["artifacts"])) + output = dict(cast(Mapping[str, Any], cfg["output"])) + resolved_run_tag = resolve_run_tag( + run_tag, + fallback_candidates=[resolve_official_baseline_run_tag()], + require_explicit=True, + ) + + with open(artifacts["conformal_results_path"], "rb") as results_handle: + results = cast(dict[str, Any], pickle.load(results_handle)) + group_metrics = pd.read_parquet(artifacts["group_metrics_path"]) + backtest_monthly = pd.read_parquet(artifacts["backtest_monthly_path"]) + alerts_path = Path(artifacts["backtest_alerts_path"]) + alerts = _load_alerts(alerts_path) + intervals_path = Path( + artifacts.get("intervals_path", "data/processed/conformal_intervals_mondrian.parquet") + ) + intervals_df = pd.read_parquet(intervals_path) + lgd_ead_status_path = Path("models/conformal_lgd_ead_status.json") + lgd_ead_status = ( + json.loads(lgd_ead_status_path.read_text(encoding="utf-8")) + if lgd_ead_status_path.exists() + else {"available": False, "reason": "missing_status_artifact"} + ) + + metrics_90 = results.get("metrics_90", {}) + metrics_95 = results.get("metrics_95", {}) + + coverage_90 = float(metrics_90.get("empirical_coverage", 0.0)) + coverage_95 = float(metrics_95.get("empirical_coverage", 0.0)) + avg_width_90 = float(metrics_90.get("avg_interval_width", 999.0)) + min_group_coverage_90 = float(group_metrics.get("coverage_90", pd.Series([0.0])).min()) + critical_alerts = int((alerts.get("severity", pd.Series([], dtype=str)) == "critical").sum()) + warning_alerts = int((alerts.get("severity", pd.Series([], dtype=str)) == "warning").sum()) + total_alerts = len(alerts) + + y90, lo90, hi90 = _interval_arrays(intervals_df, lower_col="pd_low_90", upper_col="pd_high_90") + y95, lo95, hi95 = _interval_arrays(intervals_df, lower_col="pd_low_95", upper_col="pd_high_95") + winkler_90 = _mean_winkler(y90, lo90, hi90, alpha=0.10) + winkler_95 = _mean_winkler(y95, lo95, hi95, alpha=0.05) + mapie_mwi_90 = _mapie_mwi_cross_check( + y_true=y90, + lower=lo90, + upper=hi90, + winkler_score=winkler_90, + confidence_level=0.90, + ) + violation_rate_90 = _violation_rate(y90, lo90, hi90) + violation_rate_95 = _violation_rate(y95, lo95, hi95) + + metrics = { + "coverage_90": coverage_90, + "coverage_95": coverage_95, + "avg_width_90": avg_width_90, + "min_group_coverage_90": min_group_coverage_90, + "critical_alerts": float(critical_alerts), + "warning_alerts": float(warning_alerts), + "total_alerts": float(total_alerts), + "winkler_90": winkler_90, + "winkler_95": winkler_95, + } + checks = _policy_checks(policy, metrics) + checks_df = pd.DataFrame(checks) + winkler_90_check = next(check for check in checks if check["metric"] == "winkler_90") + winkler_90_raw_pass = bool(winkler_90_check.get("raw_passed", False)) + winkler_90_policy_pass = bool(winkler_90_check.get("passed", False)) + winkler_90_policy_mode = str(winkler_90_check.get("policy_mode", "strict")) + winkler_90_compensated_pass = bool(winkler_90_check.get("compensated_passed", False)) + compensated_winkler_90_max = _safe_float(winkler_90_check.get("compensated_threshold", np.nan)) evaluation = _evaluate_check_frame(checks_df) strict_overall_pass = bool(evaluation["strict_overall_pass"]) @@ -382,11 +460,7 @@ def _evaluate_check_frame(frame: pd.DataFrame) -> dict[str, Any]: methodological_status = str(evaluation["methodological_justification_status"]) overall_pass = gate_overall_pass - latest_month = ( - backtest_monthly.sort_values("month").iloc[-1]["month"] - if not backtest_monthly.empty - else None - ) + latest_month = _latest_backtest_month(backtest_monthly) out_status = { "overall_pass": overall_pass, diff --git a/src/evaluation/backtesting.py b/src/evaluation/backtesting.py index 5d051c7..9978662 100644 --- a/src/evaluation/backtesting.py +++ b/src/evaluation/backtesting.py @@ -34,11 +34,13 @@ def cohort_analysis( for cohort, group in df.groupby(cohort_col): if len(group) < 50: continue - metrics = classification_metrics( - group[y_true_col].values, - group[y_prob_col].values, + metrics: dict[str, object] = dict( + classification_metrics( + group[y_true_col].values, + group[y_prob_col].values, + ) ) - metrics["cohort"] = cohort + metrics["cohort"] = str(cohort) metrics["n_loans"] = len(group) metrics["default_rate"] = group[y_true_col].mean() results.append(metrics) @@ -502,19 +504,19 @@ def drift_monitoring_report( if not rows: return pd.DataFrame( - columns=[ - "feature", - "train_n", - "test_n", - "psi", - "ks_statistic", - "ks_pvalue", - "cvm_statistic", - "cvm_pvalue", - "pass_psi", - "pass_ks", - "pass_cvm", - ] + { + "feature": pd.Series(dtype="object"), + "train_n": pd.Series(dtype="int64"), + "test_n": pd.Series(dtype="int64"), + "psi": pd.Series(dtype="float64"), + "ks_statistic": pd.Series(dtype="float64"), + "ks_pvalue": pd.Series(dtype="float64"), + "cvm_statistic": pd.Series(dtype="float64"), + "cvm_pvalue": pd.Series(dtype="float64"), + "pass_psi": pd.Series(dtype="bool"), + "pass_ks": pd.Series(dtype="bool"), + "pass_cvm": pd.Series(dtype="bool"), + } ) out = pd.DataFrame(rows).sort_values("psi", ascending=False).reset_index(drop=True) return out @@ -566,8 +568,8 @@ def filter_high_psi_features( ) psi_table = pd.DataFrame(psi_records).sort_values("psi", ascending=False).reset_index(drop=True) - stable = [r["feature"] for r in psi_records if r["stable"]] - drifted = [r["feature"] for r in psi_records if not r["stable"]] + stable = [str(r["feature"]) for r in psi_records if r["stable"]] + drifted = [str(r["feature"]) for r in psi_records if not r["stable"]] if drifted: logger.warning( diff --git a/src/evaluation/explainability.py b/src/evaluation/explainability.py index 7b5b489..ee2b6f8 100644 --- a/src/evaluation/explainability.py +++ b/src/evaluation/explainability.py @@ -157,14 +157,14 @@ def pairwise_shap_redundancy( ) if not rows: return pd.DataFrame( - columns=[ - "feature_a", - "feature_b", - "shap_spearman", - "value_spearman", - "redundancy_flag", - "relation_type", - ] + { + "feature_a": pd.Series(dtype="object"), + "feature_b": pd.Series(dtype="object"), + "shap_spearman": pd.Series(dtype="float64"), + "value_spearman": pd.Series(dtype="float64"), + "redundancy_flag": pd.Series(dtype="bool"), + "relation_type": pd.Series(dtype="object"), + } ) return ( pd.DataFrame(rows) diff --git a/src/evaluation/fairness.py b/src/evaluation/fairness.py index 70b67cd..e5d68fc 100644 --- a/src/evaluation/fairness.py +++ b/src/evaluation/fairness.py @@ -45,8 +45,8 @@ def demographic_parity_difference( rates = list(group_rates.values()) dpd = max(rates) - min(rates) - max_group = max(group_rates, key=group_rates.get) # type: ignore[arg-type] - min_group = min(group_rates, key=group_rates.get) # type: ignore[arg-type] + max_group = max(group_rates, key=lambda key: group_rates[key]) + min_group = min(group_rates, key=lambda key: group_rates[key]) return { "dpd": dpd, @@ -310,18 +310,18 @@ def fairness_threshold_frontier( ) if not rows: return pd.DataFrame( - columns=[ - "attribute", - "threshold", - "is_primary_threshold", - "dpd", - "eo_gap", - "dir", - "passed_dpd", - "passed_eo", - "passed_dir", - "passed_all", - ] + { + "attribute": pd.Series(dtype="object"), + "threshold": pd.Series(dtype="float64"), + "is_primary_threshold": pd.Series(dtype="bool"), + "dpd": pd.Series(dtype="float64"), + "eo_gap": pd.Series(dtype="float64"), + "dir": pd.Series(dtype="float64"), + "passed_dpd": pd.Series(dtype="bool"), + "passed_eo": pd.Series(dtype="bool"), + "passed_dir": pd.Series(dtype="bool"), + "passed_all": pd.Series(dtype="bool"), + } ) return pd.DataFrame(rows).sort_values(["attribute", "threshold"]).reset_index(drop=True) diff --git a/src/evaluation/model_shift.py b/src/evaluation/model_shift.py index c120ddc..b9f9d57 100644 --- a/src/evaluation/model_shift.py +++ b/src/evaluation/model_shift.py @@ -20,79 +20,124 @@ def interpret_model_shift( calibration_gap_delta_max: float, ) -> dict[str, Any]: """Distinguish structural shift from predictive degradation.""" - structural_level = "none" + structural_level = _structural_shift_level( + c2st_auc=c2st_auc, + score_psi=score_psi, + distribution_warning_ratio=distribution_warning_ratio, + score_psi_max=score_psi_max, + ) + predictive_level = _predictive_degradation_level( + auc_delta=auc_delta, + brier_increase=brier_increase, + calibration_gap_delta=calibration_gap_delta, + auc_delta_max=auc_delta_max, + brier_increase_max=brier_increase_max, + calibration_gap_delta_max=calibration_gap_delta_max, + ) + shift_type = _shift_type(structural_level, predictive_level) + + return { + "shift_type": shift_type, + "structural_shift_level": structural_level, + "predictive_degradation_level": predictive_level, + "governance_posture": _governance_posture(shift_type, predictive_level), + "c2st_materiality": str(c2st_materiality), + "pvalue_interpretation": _pvalue_note(shift_type), + } + + +def _structural_shift_level( + *, + c2st_auc: float, + score_psi: float, + distribution_warning_ratio: float, + score_psi_max: float, +) -> str: if c2st_auc >= 0.70 or score_psi >= max(score_psi_max * 1.5, 0.20): - structural_level = "severe" - elif c2st_auc >= 0.60 or score_psi >= score_psi_max: - structural_level = "high" - elif c2st_auc >= 0.55 or score_psi >= max(score_psi_max * 0.7, 0.10): - structural_level = "moderate" - elif c2st_auc >= 0.52 or distribution_warning_ratio > 0.05: - structural_level = "low" - - predictive_level = "none" - if ( - auc_delta >= auc_delta_max * 1.5 - or brier_increase >= brier_increase_max * 1.5 - or calibration_gap_delta >= calibration_gap_delta_max * 1.5 - ): - predictive_level = "severe" - elif ( - auc_delta > auc_delta_max - or brier_increase > brier_increase_max - or calibration_gap_delta > calibration_gap_delta_max - ): - predictive_level = "high" - elif ( - auc_delta >= auc_delta_max * 0.75 - or brier_increase >= brier_increase_max * 0.75 - or calibration_gap_delta >= calibration_gap_delta_max * 0.75 - ): - predictive_level = "moderate" - elif auc_delta > 0.0 or brier_increase > 0.0 or calibration_gap_delta > 0.0: - predictive_level = "low" + return "severe" + if c2st_auc >= 0.60 or score_psi >= score_psi_max: + return "high" + if c2st_auc >= 0.55 or score_psi >= max(score_psi_max * 0.7, 0.10): + return "moderate" + if c2st_auc >= 0.52 or distribution_warning_ratio > 0.05: + return "low" + return "none" + + +def _predictive_degradation_level( + *, + auc_delta: float, + brier_increase: float, + calibration_gap_delta: float, + auc_delta_max: float, + brier_increase_max: float, + calibration_gap_delta_max: float, +) -> str: + metrics = ( + (auc_delta, auc_delta_max), + (brier_increase, brier_increase_max), + (calibration_gap_delta, calibration_gap_delta_max), + ) + if _any_metric_crosses(metrics, multiplier=1.5, strict=False): + return "severe" + if _any_metric_crosses(metrics, multiplier=1.0, strict=True): + return "high" + if _any_metric_crosses(metrics, multiplier=0.75, strict=False): + return "moderate" + if any(value > 0.0 for value, _ in metrics): + return "low" + return "none" + + +def _any_metric_crosses( + metrics: tuple[tuple[float, float], ...], + *, + multiplier: float, + strict: bool, +) -> bool: + for value, limit in metrics: + threshold = limit * multiplier + crosses = value > threshold if strict else value >= threshold + if crosses: + return True + return False + +def _shift_type(structural_level: str, predictive_level: str) -> str: if structural_level != "none" and predictive_level in {"none", "low"}: - shift_type = "structural_shift_only" - elif structural_level == "none" and predictive_level != "none": - shift_type = "predictive_degradation" - elif structural_level != "none" and predictive_level != "none": - shift_type = "mixed_shift" - else: - shift_type = "stable" + return "structural_shift_only" + if structural_level == "none" and predictive_level != "none": + return "predictive_degradation" + if structural_level != "none" and predictive_level != "none": + return "mixed_shift" + return "stable" + +def _governance_posture(shift_type: str, predictive_level: str) -> str: if shift_type == "mixed_shift" or predictive_level in {"high", "severe"}: - governance_posture = "candidate_gate" - elif shift_type == "structural_shift_only": - governance_posture = "warning_only" - else: - governance_posture = "monitor" + return "candidate_gate" + if shift_type == "structural_shift_only": + return "warning_only" + return "monitor" - pvalue_note = ( - "Distribution p-values are informative, but they can trigger on small effects at large N; " - "the governance posture should be anchored on materiality, predictive loss, and C2ST/score shift." - ) + +def _pvalue_note(shift_type: str) -> str: if shift_type == "structural_shift_only": - pvalue_note = ( + return ( "KS/CvM or C2ST detect structural population change without clear predictive degradation. " "Treat this as representativeness pressure and monitoring, not an automatic retraining gate." ) - elif shift_type == "predictive_degradation": - pvalue_note = ( + if shift_type == "predictive_degradation": + return ( "Predictive degradation is visible even without strong structural shift. " "Operational metrics dominate p-value interpretation in this case." ) - elif shift_type == "mixed_shift": - pvalue_note = ( + if shift_type == "mixed_shift": + return ( "Both structural shift and predictive degradation are present. " "This combination deserves the strongest governance posture." ) - - return { - "shift_type": shift_type, - "structural_shift_level": structural_level, - "predictive_degradation_level": predictive_level, - "governance_posture": governance_posture, - "c2st_materiality": str(c2st_materiality), - "pvalue_interpretation": pvalue_note, - } + return ( + "Distribution p-values are informative, but they can trigger on small effects at large N; " + "the governance posture should be anchored on materiality, predictive loss, and C2ST/score shift." + ) diff --git a/src/features/feature_config_io.py b/src/features/feature_config_io.py index 7a54b91..0dd38f2 100644 --- a/src/features/feature_config_io.py +++ b/src/features/feature_config_io.py @@ -208,8 +208,8 @@ def _frame_to_config(frame: pd.DataFrame) -> dict[str, Any]: cfg[str(section)] = [json.loads(raw) for raw in group["value_json"]] elif kind == "dict": cfg[str(section)] = { - str(row.key): json.loads(str(row.value_json)) - for row in group.itertuples(index=False) + str(row["key"]): json.loads(str(row["value_json"])) + for row in group.to_dict("records") } elif kind == "scalar": if len(group) != 1: diff --git a/src/features/feature_engineering.py b/src/features/feature_engineering.py index df5ec69..5918ac9 100644 --- a/src/features/feature_engineering.py +++ b/src/features/feature_engineering.py @@ -797,7 +797,8 @@ def save_feature_artifacts( feature_config=feature_config, ) atomic_write_parquet(manifest, out_dir / "feature_manifest_v2.parquet", index=False) - atomic_write_text( - out_dir / "feature_manifest_v2.json", manifest.to_json(orient="records", indent=2) - ) + manifest_json = manifest.to_json(orient="records", indent=2) + if manifest_json is None: + raise RuntimeError("pandas returned no feature manifest JSON payload") + atomic_write_text(out_dir / "feature_manifest_v2.json", manifest_json) logger.info("Saved canonical feature artifacts to {}", out_dir) diff --git a/src/features/tabprep_challenger.py b/src/features/tabprep_challenger.py index 104ac27..6d65c55 100644 --- a/src/features/tabprep_challenger.py +++ b/src/features/tabprep_challenger.py @@ -14,7 +14,7 @@ import itertools from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field -from typing import Any +from typing import Any, cast import numpy as np import pandas as pd @@ -486,14 +486,14 @@ def _select_arithmetic_specs(self, frame: pd.DataFrame, y: pd.Series) -> list[Fe triple_candidates: list[FeatureSpec] = [] triple_base = numeric[: min(20, len(numeric))] - for sources in itertools.combinations(triple_base, 3): + for triple_sources in itertools.combinations(triple_base, 3): for operation in ("product3", "sum3", "ratio_sum"): - name = _make_feature_name("tp_ar", operation, sources) + name = _make_feature_name("tp_ar", operation, triple_sources) spec = FeatureSpec( name=name, generator="arithmetic", operation=operation, - source_features=sources, + source_features=triple_sources, score=0.0, selected_rank=0, ) @@ -953,6 +953,7 @@ def _fit_groupby_state( if operation == "pct_rank": state.global_quantiles = _quantile_grid(values) for group, group_values in tmp.groupby("group", observed=True)["value"]: + group_values = cast(pd.Series, group_values) if len(group_values) >= int(min_support): state.group_quantiles[str(group)] = _quantile_grid(group_values) return state diff --git a/src/models/calibration.py b/src/models/calibration.py index fa821f2..06266cc 100644 --- a/src/models/calibration.py +++ b/src/models/calibration.py @@ -29,7 +29,8 @@ def transform(self, scores: np.ndarray) -> np.ndarray: scores_arr = np.clip(np.asarray(scores, dtype=float), 1e-6, 1.0 - 1e-6) logits = np.log(scores_arr / (1.0 - scores_arr)) shifted = 1.0 / (1.0 + np.exp(-(logits + self.delta))) - return cast(np.ndarray, np.clip(np.asarray(shifted, dtype=float), 0.0, 1.0)) + clipped: np.ndarray = np.clip(np.asarray(shifted, dtype=float), 0.0, 1.0) + return clipped def predict(self, scores: np.ndarray) -> np.ndarray: return self.transform(scores) @@ -51,11 +52,12 @@ def __init__(self, temperature: float = 1.0) -> None: @staticmethod def _logit(scores: np.ndarray) -> np.ndarray: clipped = np.clip(np.asarray(scores, dtype=float).reshape(-1), 1e-6, 1.0 - 1e-6) - return np.log(clipped / (1.0 - clipped)) + logits: np.ndarray = np.log(clipped / (1.0 - clipped)) + return logits @staticmethod def _sigmoid(logits: np.ndarray) -> np.ndarray: - return 1.0 / (1.0 + np.exp(-logits)) + return cast(np.ndarray, 1.0 / (1.0 + np.exp(-logits))) def fit(self, y_prob_raw: np.ndarray, y_true: np.ndarray) -> TemperatureScalingCalibrator: logits = self._logit(y_prob_raw) @@ -76,10 +78,8 @@ def predict(self, y_prob_raw: np.ndarray) -> np.ndarray: if not self._is_fitted: raise RuntimeError("TemperatureScalingCalibrator is not fitted.") logits = self._logit(y_prob_raw) - return cast( - np.ndarray, - np.clip(self._sigmoid(logits / max(self.temperature, 1e-3)), 0.0, 1.0), - ) + clipped: np.ndarray = np.clip(self._sigmoid(logits / max(self.temperature, 1e-3)), 0.0, 1.0) + return clipped class QuadraticLogitCalibrator: @@ -130,7 +130,11 @@ def adaptive_calibration_error(y_true: np.ndarray, y_prob: np.ndarray, n_bins: i order = np.argsort(p_arr, kind="mergesort") y_sorted = y_arr[order] p_sorted = p_arr[order] - bins = [chunk for chunk in np.array_split(np.arange(len(y_sorted)), max(1, int(n_bins))) if len(chunk)] + bins = [ + chunk + for chunk in np.array_split(np.arange(len(y_sorted)), max(1, int(n_bins))) + if len(chunk) + ] ace = 0.0 for idx in bins: bin_acc = float(y_sorted[idx].mean()) diff --git a/src/models/conformal_adapters.py b/src/models/conformal_adapters.py index 293c59b..8918709 100644 --- a/src/models/conformal_adapters.py +++ b/src/models/conformal_adapters.py @@ -29,7 +29,7 @@ import numpy as np import pandas as pd -from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin # Canonical module path that pickled instances must report. _PICKLE_MODULE = "src.models.conformal" @@ -60,7 +60,7 @@ def predict(self, X: Any) -> np.ndarray: return apply_probability_calibrator(self.calibrator, raw) -class PrefitClassifierAdapter(BaseEstimator): +class PrefitClassifierAdapter(ClassifierMixin, BaseEstimator): """Small sklearn-style adapter for prefit classifiers inside MAPIE checks.""" def __init__(self, classifier: Any, n_features_in: int | None = None) -> None: @@ -80,9 +80,7 @@ def _is_minimal_probe(self, X: pd.DataFrame) -> bool: if X.shape[0] != 1 or X.shape[1] != self.n_features_in_: return False numeric = X.apply(pd.to_numeric, errors="coerce") - return bool( - np.isfinite(numeric.to_numpy()).all() and np.allclose(numeric.to_numpy(), 0.0) - ) + return bool(np.isfinite(numeric.to_numpy()).all() and np.allclose(numeric.to_numpy(), 0.0)) def predict(self, X: Any) -> np.ndarray: X_df = pd.DataFrame(X) if not isinstance(X, pd.DataFrame) else X diff --git a/src/models/conformal_alpha_grid.py b/src/models/conformal_alpha_grid.py new file mode 100644 index 0000000..cad3e6a --- /dev/null +++ b/src/models/conformal_alpha_grid.py @@ -0,0 +1,211 @@ +"""Exact alpha-grid replay for a frozen Mondrian conformal recipe.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + +from src.models.conformal import ( + build_mondrian_partition_labels, + create_pd_intervals_mondrian_from_predictions, +) +from src.models.conformal_tuning import apply_group_multipliers, build_group_temporal_segments + + +@dataclass(frozen=True) +class FrozenConformalRecipe: + """Selected interval design and holdout-learned widening policy.""" + + partition: str + partition_probability_source: str + n_score_bins: int + fallback_mode: str + score_scale_family: str + min_group_size: int + reference_target_alpha: float + reference_used_alpha: float + calibration_fraction: float + tuning_holdout_ratio: float + tuning_random_state: int + group_multipliers: dict[str, float] + temporal_segment_multipliers: dict[str, float] + temporal_segment_freq: str + global_rebalance_factor: float + + @classmethod + def from_results_payload(cls, payload: dict[str, Any]) -> FrozenConformalRecipe: + """Build a replay recipe from ``conformal_results_mondrian.pkl``.""" + selected = payload["tuning_90_best"] + split = payload["calibration_split"] + global_rebalance = payload.get("global_rebalance", {}) or {} + factor = ( + float(global_rebalance.get("factor", 1.0)) + if bool(global_rebalance.get("applied", False)) + else 1.0 + ) + recipe = cls( + partition=str(selected["partition"]), + partition_probability_source=str(selected["partition_probability_source"]), + n_score_bins=int(selected["n_score_bins"]), + fallback_mode=str(selected["fallback_mode"]), + score_scale_family=str(selected["score_scale_family"]), + min_group_size=int(selected["min_group_size"]), + reference_target_alpha=float(selected["alpha_target_90"]), + reference_used_alpha=float(selected["alpha_used_90"]), + calibration_fraction=float(split["calibration_fraction"]), + tuning_holdout_ratio=float(split["holdout_ratio"]), + tuning_random_state=int(split["random_state"]), + group_multipliers={ + str(key): float(value) + for key, value in (payload.get("group_coverage_multipliers", {}) or {}).items() + }, + temporal_segment_multipliers={ + str(key): float(value) + for key, value in (payload.get("temporal_segment_multipliers", {}) or {}).items() + }, + temporal_segment_freq=str(payload.get("temporal_segment_freq", "Q")), + global_rebalance_factor=factor, + ) + recipe.validate() + return recipe + + def validate(self) -> None: + """Reject recipe settings that could narrow a nominal interval silently.""" + if not 0.0 < self.reference_target_alpha < 1.0: + raise ValueError("reference_target_alpha must lie in (0, 1).") + if not 0.0 < self.reference_used_alpha <= self.reference_target_alpha: + raise ValueError( + "reference_used_alpha must be positive and no larger than the target alpha." + ) + multipliers = [ + *self.group_multipliers.values(), + *self.temporal_segment_multipliers.values(), + self.global_rebalance_factor, + ] + if any(value < 1.0 for value in multipliers): + raise ValueError("Frozen alpha-grid replay only supports widening adjustments.") + + def used_alpha(self, target_alpha: float) -> float: + """Apply the frozen conservative alpha ratio selected at the reference level.""" + target = float(target_alpha) + if not 0.0 < target < 1.0: + raise ValueError("target_alpha must lie in (0, 1).") + ratio = self.reference_used_alpha / self.reference_target_alpha + return target * ratio + + +@dataclass(frozen=True) +class ExactAlphaIntervals: + """One exact conformal interval vector and its replay metadata.""" + + target_alpha: float + used_alpha: float + point: np.ndarray + low: np.ndarray + high: np.ndarray + partition_labels: pd.Series + partition_metadata: dict[str, Any] + diagnostics: dict[str, Any] + + +def alpha_column_token(alpha: float) -> str: + """Return a stable column-safe token such as ``0p010``.""" + return f"{float(alpha):.3f}".replace(".", "p") + + +def alpha_interval_columns(alpha: float) -> tuple[str, str]: + """Return the low/high column names for an exact alpha-grid artifact.""" + token = alpha_column_token(alpha) + return f"pd_low_alpha_{token}", f"pd_high_alpha_{token}" + + +def _scale_around_prediction( + point: np.ndarray, + intervals: np.ndarray, + factor: float, +) -> np.ndarray: + radius = np.maximum(point - intervals[:, 0], intervals[:, 1] - point) + return np.column_stack( + [ + np.clip(point - radius * factor, 0.0, 1.0), + np.clip(point + radius * factor, 0.0, 1.0), + ] + ) + + +def compute_exact_alpha_intervals( + *, + recipe: FrozenConformalRecipe, + target_alpha: float, + y_cal: pd.Series | np.ndarray, + interval_probability_cal: np.ndarray, + interval_probability_eval: np.ndarray, + partition_probability_cal: np.ndarray, + partition_probability_eval: np.ndarray, + base_groups_cal: pd.Series | np.ndarray, + base_groups_eval: pd.Series | np.ndarray, + issue_dates_eval: pd.Series | np.ndarray | None = None, +) -> ExactAlphaIntervals: + """Recompute one alpha exactly under a frozen partition and widening recipe.""" + group_cal, group_eval, partition_metadata = build_mondrian_partition_labels( + y_prob_cal=partition_probability_cal, + y_prob_eval=partition_probability_eval, + partition=recipe.partition, + base_groups_cal=base_groups_cal, + base_groups_eval=base_groups_eval, + n_score_bins=recipe.n_score_bins, + min_group_size=recipe.min_group_size, + fallback_mode=recipe.fallback_mode, + ) + used_alpha = recipe.used_alpha(target_alpha) + point, intervals, diagnostics = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=interval_probability_cal, + y_test_pred=interval_probability_eval, + y_cal=y_cal, + group_cal=group_cal, + group_test=group_eval, + alpha=used_alpha, + min_group_size=recipe.min_group_size, + score_scale_family=recipe.score_scale_family, + log_summary=False, + ) + if recipe.group_multipliers: + intervals = apply_group_multipliers( + point, + intervals, + group_eval, + recipe.group_multipliers, + ) + if recipe.temporal_segment_multipliers: + if issue_dates_eval is None: + raise ValueError("issue_dates_eval is required by the frozen temporal multipliers.") + temporal_segments = build_group_temporal_segments( + group_eval, + issue_dates_eval, + freq=recipe.temporal_segment_freq, + ) + intervals = apply_group_multipliers( + point, + intervals, + temporal_segments, + recipe.temporal_segment_multipliers, + ) + if not np.isclose(recipe.global_rebalance_factor, 1.0): + intervals = _scale_around_prediction( + point, + intervals, + recipe.global_rebalance_factor, + ) + return ExactAlphaIntervals( + target_alpha=float(target_alpha), + used_alpha=used_alpha, + point=point, + low=intervals[:, 0], + high=intervals[:, 1], + partition_labels=group_eval, + partition_metadata=partition_metadata, + diagnostics=diagnostics, + ) diff --git a/src/models/conformal_tuning.py b/src/models/conformal_tuning.py index 7ac6d99..e32226f 100644 --- a/src/models/conformal_tuning.py +++ b/src/models/conformal_tuning.py @@ -10,6 +10,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import Any, cast import numpy as np @@ -542,6 +543,215 @@ def temporal_stability_summary( } +@dataclass(frozen=True) +class _ShrinkContext: + y_true: np.ndarray + y_pred: np.ndarray + base_intervals: np.ndarray + groups: pd.Series + temporal_segments: pd.Series | None + issue_dates: pd.Series | np.ndarray | None + target_coverage: float + min_group_coverage_target: float + max_monthly_gap_target: float | None + alpha: float + + +@dataclass(frozen=True) +class _ShrinkCandidate: + scope: str + key: str + factor: float + accepted: bool + intervals: np.ndarray + group_factors: dict[str, float] + temporal_factors: dict[str, float] + metrics: dict[str, float] + + +def _active_widening_factors(factors: dict[str, float] | None) -> dict[str, float]: + return {str(k): float(v) for k, v in (factors or {}).items() if float(v) > 1.0} + + +def _apply_shrink_factors( + context: _ShrinkContext, + group_factors: dict[str, float], + temporal_factors: dict[str, float], +) -> np.ndarray: + intervals = context.base_intervals.copy() + if group_factors: + intervals = apply_group_multipliers( + context.y_pred, + intervals, + context.groups, + group_factors, + ) + if temporal_factors and context.temporal_segments is not None: + intervals = apply_group_multipliers( + context.y_pred, + intervals, + context.temporal_segments, + temporal_factors, + ) + return intervals + + +def _shrink_metrics(context: _ShrinkContext, intervals: np.ndarray) -> dict[str, float]: + temporal = temporal_stability_summary( + context.y_true, + intervals, + context.issue_dates, + target_coverage=context.target_coverage, + freq="M", + ) + return { + "coverage": empirical_interval_coverage(context.y_true, intervals), + "min_group_coverage": min_group_interval_coverage( + context.y_true, + intervals, + context.groups, + ), + "avg_width": average_interval_width(intervals), + "winkler_90": mean_winkler_score(context.y_true, intervals, alpha=context.alpha), + "max_monthly_gap": float(temporal["max_monthly_gap"]), + "stability_over_time": float(temporal["stability_over_time"]), + } + + +def _shrink_constraints_ok(context: _ShrinkContext, metrics: dict[str, float]) -> bool: + if float(metrics["coverage"]) < context.target_coverage: + return False + if float(metrics["min_group_coverage"]) < context.min_group_coverage_target: + return False + return not ( + context.max_monthly_gap_target is not None + and np.isfinite(context.max_monthly_gap_target) + and float(metrics["max_monthly_gap"]) > context.max_monthly_gap_target + ) + + +def _next_lower_factor(value: float, grid: tuple[float, ...]) -> float | None: + ordered = sorted({round(float(x), 6) for x in grid if float(x) <= float(value) + 1e-9}) + current = round(float(value), 6) + if current not in ordered: + ordered.append(current) + ordered = sorted(set(ordered)) + idx = ordered.index(current) + if idx == 0: + return None + return float(ordered[idx - 1]) + + +def _with_reduced_factor( + factors: dict[str, float], + key: str, + next_factor: float, +) -> dict[str, float]: + reduced = dict(factors) + if next_factor <= 1.0: + reduced.pop(key, None) + else: + reduced[key] = float(next_factor) + return reduced + + +def _evaluate_shrink_candidate( + context: _ShrinkContext, + *, + scope: str, + key: str, + factor: float, + group_factors: dict[str, float], + temporal_factors: dict[str, float], +) -> _ShrinkCandidate: + intervals = _apply_shrink_factors(context, group_factors, temporal_factors) + metrics = _shrink_metrics(context, intervals) + return _ShrinkCandidate( + scope=scope, + key=key, + factor=float(factor), + accepted=_shrink_constraints_ok(context, metrics), + intervals=intervals, + group_factors=dict(group_factors), + temporal_factors=dict(temporal_factors), + metrics=metrics, + ) + + +def _build_shrink_attempts( + context: _ShrinkContext, + *, + scope: str, + group_factors: dict[str, float], + temporal_factors: dict[str, float], + multiplier_grid: tuple[float, ...], +) -> list[_ShrinkCandidate]: + active_factors = group_factors if scope == "group" else temporal_factors + candidates: list[_ShrinkCandidate] = [] + for key, value in list(active_factors.items()): + next_factor = _next_lower_factor(value, multiplier_grid) + if next_factor is None: + continue + trial_group = ( + _with_reduced_factor(group_factors, key, next_factor) + if scope == "group" + else dict(group_factors) + ) + trial_temporal = ( + _with_reduced_factor(temporal_factors, key, next_factor) + if scope == "temporal" + else dict(temporal_factors) + ) + candidates.append( + _evaluate_shrink_candidate( + context, + scope=scope, + key=key, + factor=next_factor, + group_factors=trial_group, + temporal_factors=trial_temporal, + ) + ) + return candidates + + +def _shrink_candidate_is_better( + candidate: _ShrinkCandidate, + best_candidate: _ShrinkCandidate | None, +) -> bool: + if not candidate.accepted: + return False + if best_candidate is None: + return True + candidate_width = float(candidate.metrics["avg_width"]) + best_width = float(best_candidate.metrics["avg_width"]) + if candidate_width < best_width: + return True + return bool( + np.isclose(candidate_width, best_width) + and float(candidate.metrics["winkler_90"]) < float(best_candidate.metrics["winkler_90"]) + ) + + +def _shrink_report_row( + *, + stage: str, + factor_scope: str, + factor_key: str, + candidate_factor: float, + accepted: bool, + metrics: dict[str, float], +) -> dict[str, Any]: + return { + "stage": stage, + "factor_scope": factor_scope, + "factor_key": factor_key, + "candidate_factor": candidate_factor, + "accepted": accepted, + **metrics, + } + + def shrink_group_multipliers( *, y_true: np.ndarray, @@ -560,222 +770,114 @@ def shrink_group_multipliers( temporal_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), ) -> tuple[np.ndarray, dict[str, float], dict[str, float], pd.DataFrame]: """Greedily shrink learned widening factors while preserving constraints.""" - group_factors_cur = { - str(k): float(v) for k, v in (group_factors or {}).items() if float(v) > 1.0 - } - temporal_factors_cur = { - str(k): float(v) for k, v in (temporal_factors or {}).items() if float(v) > 1.0 - } - base = np.asarray(base_intervals, dtype=float) - y_pred_arr = np.asarray(y_pred, dtype=float) - y_true_arr = np.asarray(y_true, dtype=float) - group_series = pd.Series(groups).fillna("UNKNOWN").astype(str).reset_index(drop=True) + group_factors_cur = _active_widening_factors(group_factors) + temporal_factors_cur = _active_widening_factors(temporal_factors) temporal_series = ( pd.Series(temporal_segments).fillna("UNKNOWN").astype(str).reset_index(drop=True) if temporal_segments is not None else None ) + context = _ShrinkContext( + y_true=np.asarray(y_true, dtype=float), + y_pred=np.asarray(y_pred, dtype=float), + base_intervals=np.asarray(base_intervals, dtype=float), + groups=pd.Series(groups).fillna("UNKNOWN").astype(str).reset_index(drop=True), + temporal_segments=temporal_series, + issue_dates=issue_dates, + target_coverage=float(target_coverage), + min_group_coverage_target=float(min_group_coverage_target), + max_monthly_gap_target=max_monthly_gap_target, + alpha=float(alpha), + ) - def _apply_all( - gf: dict[str, float], - tf: dict[str, float], - ) -> np.ndarray: - intervals = base.copy() - if gf: - intervals = apply_group_multipliers(y_pred_arr, intervals, group_series, gf) - if tf and temporal_series is not None: - intervals = apply_group_multipliers(y_pred_arr, intervals, temporal_series, tf) - return intervals - - def _metrics(intervals: np.ndarray) -> dict[str, float]: - temporal = temporal_stability_summary( - y_true_arr, - intervals, - issue_dates, - target_coverage=float(target_coverage), - freq="M", - ) - return { - "coverage": empirical_interval_coverage(y_true_arr, intervals), - "min_group_coverage": min_group_interval_coverage(y_true_arr, intervals, group_series), - "avg_width": average_interval_width(intervals), - "winkler_90": mean_winkler_score(y_true_arr, intervals, alpha=alpha), - "max_monthly_gap": float(temporal["max_monthly_gap"]), - "stability_over_time": float(temporal["stability_over_time"]), - } - - def _constraints_ok(metrics: dict[str, float]) -> bool: - if float(metrics["coverage"]) < float(target_coverage): - return False - if float(metrics["min_group_coverage"]) < float(min_group_coverage_target): - return False - return not ( - max_monthly_gap_target is not None - and np.isfinite(max_monthly_gap_target) - and float(metrics["max_monthly_gap"]) > float(max_monthly_gap_target) - ) - - current_intervals = _apply_all(group_factors_cur, temporal_factors_cur) - current_metrics = _metrics(current_intervals) + current_intervals = _apply_shrink_factors( + context, + group_factors_cur, + temporal_factors_cur, + ) + current_metrics = _shrink_metrics(context, current_intervals) report_rows: list[dict[str, Any]] = [ - { - "stage": "initial", - "factor_scope": "all", - "factor_key": "all", - "candidate_factor": np.nan, - "accepted": True, - **current_metrics, - } + _shrink_report_row( + stage="initial", + factor_scope="all", + factor_key="all", + candidate_factor=np.nan, + accepted=True, + metrics=current_metrics, + ) ] - if not _constraints_ok(current_metrics): + if not _shrink_constraints_ok(context, current_metrics): report_rows.append( - { - "stage": "initial_infeasible", - "factor_scope": "all", - "factor_key": "all", - "candidate_factor": np.nan, - "accepted": False, - **current_metrics, - } + _shrink_report_row( + stage="initial_infeasible", + factor_scope="all", + factor_key="all", + candidate_factor=np.nan, + accepted=False, + metrics=current_metrics, + ) ) return current_intervals, group_factors_cur, temporal_factors_cur, pd.DataFrame(report_rows) - def _next_lower(value: float, grid: tuple[float, ...]) -> float | None: - ordered = sorted({round(float(x), 6) for x in grid if float(x) <= float(value) + 1e-9}) - current = round(float(value), 6) - if current not in ordered: - ordered.append(current) - ordered = sorted(set(ordered)) - idx = ordered.index(current) - if idx == 0: - return None - return float(ordered[idx - 1]) - while True: - best_candidate: dict[str, Any] | None = None - - for key, value in list(group_factors_cur.items()): - next_factor = _next_lower(value, group_multiplier_grid) - if next_factor is None: - continue - trial_group = dict(group_factors_cur) - if next_factor <= 1.0: - trial_group.pop(key, None) - else: - trial_group[key] = next_factor - trial_intervals = _apply_all(trial_group, temporal_factors_cur) - trial_metrics = _metrics(trial_intervals) - accepted = _constraints_ok(trial_metrics) - candidate: dict[str, Any] = { - "scope": "group", - "key": key, - "factor": next_factor, - "accepted": accepted, - "intervals": trial_intervals, - "group_factors": trial_group, - "temporal_factors": dict(temporal_factors_cur), - "metrics": trial_metrics, - } - if accepted and ( - best_candidate is None - or float(candidate["metrics"]["avg_width"]) - < float(best_candidate["metrics"]["avg_width"]) - or ( - np.isclose( - float(candidate["metrics"]["avg_width"]), - float(best_candidate["metrics"]["avg_width"]), - ) - and float(candidate["metrics"]["winkler_90"]) - < float(best_candidate["metrics"]["winkler_90"]) - ) - ): - best_candidate = candidate - report_rows.append( - { - "stage": "attempt", - "factor_scope": "group", - "factor_key": key, - "candidate_factor": float(next_factor), - "accepted": bool(accepted), - **trial_metrics, - } + attempts = _build_shrink_attempts( + context, + scope="group", + group_factors=group_factors_cur, + temporal_factors=temporal_factors_cur, + multiplier_grid=group_multiplier_grid, + ) + attempts.extend( + _build_shrink_attempts( + context, + scope="temporal", + group_factors=group_factors_cur, + temporal_factors=temporal_factors_cur, + multiplier_grid=temporal_multiplier_grid, ) - - for key, value in list(temporal_factors_cur.items()): - next_factor = _next_lower(value, temporal_multiplier_grid) - if next_factor is None: - continue - trial_temporal = dict(temporal_factors_cur) - if next_factor <= 1.0: - trial_temporal.pop(key, None) - else: - trial_temporal[key] = next_factor - trial_intervals = _apply_all(group_factors_cur, trial_temporal) - trial_metrics = _metrics(trial_intervals) - accepted = _constraints_ok(trial_metrics) - candidate = { - "scope": "temporal", - "key": key, - "factor": next_factor, - "accepted": accepted, - "intervals": trial_intervals, - "group_factors": dict(group_factors_cur), - "temporal_factors": trial_temporal, - "metrics": trial_metrics, - } - if accepted and ( - best_candidate is None - or float(candidate["metrics"]["avg_width"]) - < float(best_candidate["metrics"]["avg_width"]) - or ( - np.isclose( - float(candidate["metrics"]["avg_width"]), - float(best_candidate["metrics"]["avg_width"]), - ) - and float(candidate["metrics"]["winkler_90"]) - < float(best_candidate["metrics"]["winkler_90"]) - ) - ): + ) + best_candidate: _ShrinkCandidate | None = None + for candidate in attempts: + if _shrink_candidate_is_better(candidate, best_candidate): best_candidate = candidate report_rows.append( - { - "stage": "attempt", - "factor_scope": "temporal", - "factor_key": key, - "candidate_factor": float(next_factor), - "accepted": bool(accepted), - **trial_metrics, - } + _shrink_report_row( + stage="attempt", + factor_scope=candidate.scope, + factor_key=candidate.key, + candidate_factor=float(candidate.factor), + accepted=bool(candidate.accepted), + metrics=candidate.metrics, + ) ) - if best_candidate is None: break - current_intervals = np.asarray(best_candidate["intervals"], dtype=float) - group_factors_cur = dict(best_candidate["group_factors"]) - temporal_factors_cur = dict(best_candidate["temporal_factors"]) - current_metrics = dict(best_candidate["metrics"]) + current_intervals = np.asarray(best_candidate.intervals, dtype=float) + group_factors_cur = dict(best_candidate.group_factors) + temporal_factors_cur = dict(best_candidate.temporal_factors) + current_metrics = dict(best_candidate.metrics) report_rows.append( - { - "stage": "accepted", - "factor_scope": str(best_candidate["scope"]), - "factor_key": str(best_candidate["key"]), - "candidate_factor": float(best_candidate["factor"]), - "accepted": True, - **current_metrics, - } + _shrink_report_row( + stage="accepted", + factor_scope=best_candidate.scope, + factor_key=best_candidate.key, + candidate_factor=float(best_candidate.factor), + accepted=True, + metrics=current_metrics, + ) ) report_rows.append( - { - "stage": "final", - "factor_scope": "all", - "factor_key": "all", - "candidate_factor": np.nan, - "accepted": True, - **current_metrics, - } + _shrink_report_row( + stage="final", + factor_scope="all", + factor_key="all", + candidate_factor=np.nan, + accepted=True, + metrics=current_metrics, + ) ) report = pd.DataFrame(report_rows) return current_intervals, group_factors_cur, temporal_factors_cur, report diff --git a/src/models/optuna_tuning.py b/src/models/optuna_tuning.py index f05996f..2c5a724 100644 --- a/src/models/optuna_tuning.py +++ b/src/models/optuna_tuning.py @@ -3,6 +3,8 @@ from __future__ import annotations import gc +from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import Any @@ -17,6 +19,16 @@ SEARCH_SPACE_VERSION = "cb_space_v2" _JOURNAL_STORAGE_PREFIXES = ("journal+file:", "journalfile:", "journal:") +CategoricalChoice = None | bool | int | float | str + + +@dataclass(frozen=True) +class _SelectedModelFit: + model: CatBoostClassifier + selected_params: dict[str, Any] + resolved_params: dict[str, Any] + validation_auc: float + best_iteration: int def _is_journal_storage_url(url: str) -> bool: @@ -33,6 +45,18 @@ def _journal_path_from_storage_url(url: str) -> str: return url +def _categorical_choices(raw: Any) -> list[CategoricalChoice]: + choices: list[CategoricalChoice] = [] + for value in list(raw): + if value is None or isinstance(value, bool | int | float | str): + choices.append(value) + else: + raise TypeError(f"Unsupported Optuna categorical choice {value!r}") + if not choices: + raise ValueError("Optuna categorical choices must not be empty") + return choices + + def resolve_optuna_study_name( study_name: str | None, *, @@ -103,629 +127,707 @@ def _build_optuna_sampler_pruner( return sampler_obj, pruner_obj -def train_catboost_tuned_optuna( - X_train: pd.DataFrame, - y_train: pd.Series, - X_val: pd.DataFrame, - y_val: pd.Series, - X_test: pd.DataFrame | None = None, - y_test: pd.Series | None = None, +def _align_feature_vector( + raw: Any, *, - cat_features: list[str] | None = None, - base_params: dict[str, Any] | None = None, - n_trials: int = 100, - sampler: str = "tpe", - pruner: str = "median", - timeout_minutes: int = 0, - n_startup_trials: int = 40, - multivariate_tpe: bool = True, - group_tpe: bool = True, - warn_independent_sampling: bool = True, - constant_liar: bool = False, - pruner_n_startup_trials: int = 20, - pruner_n_warmup_steps: int = 50, - use_pruning_callback: bool = True, - study_storage: str | None = None, - study_name: str | None = None, - load_if_exists: bool = True, - refit_full_train: bool = True, - gc_after_trial: bool = True, - storage_heartbeat_interval: int = 0, - storage_grace_period: int = 0, - sqlite_timeout_seconds: int = 60, - retry_failed_trials: int = 0, - n_jobs: int = 1, - sample_weight: np.ndarray | None = None, - eval_sample_weight: np.ndarray | None = None, - search_space_mode: str = "global", - local_refine_space: dict[str, Any] | None = None, - constraints_policy: dict[str, Any] | None = None, - search_space_version: str = SEARCH_SPACE_VERSION, - enqueue_trials: list[dict[str, Any]] | None = None, -) -> tuple[CatBoostClassifier, dict[str, Any]]: - """Tune CatBoost with Optuna and return best fitted model and metadata.""" - import optuna + feature_order: list[str], + default: float, +) -> list[float] | Any: + if not isinstance(raw, dict): + return raw + raw_by_feature = {str(feature): float(value) for feature, value in raw.items()} + return [float(raw_by_feature.get(feature, default)) for feature in feature_order] - if cat_features is None: - cat_features = [c for c in CATEGORICAL_FEATURES if c in X_train.columns] - base = _catboost_base_params(base_params) - base["verbose"] = 0 - has_monotone_constraints = bool(str(base.get("monotone_constraints", "")).strip()) - if has_monotone_constraints: - base["grow_policy"] = "SymmetricTree" - search_space_mode_resolved = str(search_space_mode or "global").strip().lower() or "global" - local_refine_space = dict(local_refine_space or {}) - if has_monotone_constraints: - local_refine_space["grow_policy"] = ["SymmetricTree"] - constraints_policy = dict(constraints_policy or {}) - enqueue_trials = list(enqueue_trials or []) - feature_order = [str(col) for col in X_train.columns] +def _normalize_feature_penalty_params( + params: dict[str, Any], + *, + feature_order: list[str], +) -> dict[str, Any]: + normalized = dict(params) + if "feature_weights" in normalized: + normalized["feature_weights"] = _align_feature_vector( + normalized["feature_weights"], + feature_order=feature_order, + default=1.0, + ) + if "first_feature_use_penalties" in normalized: + normalized["first_feature_use_penalties"] = _align_feature_vector( + normalized["first_feature_use_penalties"], + feature_order=feature_order, + default=0.0, + ) + return normalized + + +def _local_choice(trial: Any, name: str, spec: Any, default: Any) -> Any: + if spec is None: + return default + if isinstance(spec, dict): + if spec.get("choices") is not None: + return trial.suggest_categorical(name, _categorical_choices(spec["choices"])) + low = spec.get("low") + high = spec.get("high") + step = spec.get("step") + log = bool(spec.get("log", False)) + if low is None or high is None: + return default + if isinstance(low, int) and isinstance(high, int) and not log: + return trial.suggest_int(name, int(low), int(high), step=int(step or 1)) + return trial.suggest_float( + name, + float(low), + float(high), + step=None if log else (float(step) if step is not None else None), + log=log, + ) + if isinstance(spec, list): + return trial.suggest_categorical(name, _categorical_choices(spec)) + return spec - def _align_feature_vector(raw: Any, *, default: float) -> list[float] | Any: - if not isinstance(raw, dict): - return raw - raw_by_feature = {str(feature): float(value) for feature, value in raw.items()} - return [float(raw_by_feature.get(feature, default)) for feature in feature_order] - - def _normalize_feature_penalty_params(params: dict[str, Any]) -> dict[str, Any]: - normalized = dict(params) - if "feature_weights" in normalized: - normalized["feature_weights"] = _align_feature_vector( - normalized["feature_weights"], - default=1.0, - ) - if "first_feature_use_penalties" in normalized: - normalized["first_feature_use_penalties"] = _align_feature_vector( - normalized["first_feature_use_penalties"], - default=0.0, - ) - return normalized - incumbent_metrics: dict[str, float] = {} +def _apply_local_feature_priors( + trial: Any, + params: dict[str, Any], + *, + local_refine_space: dict[str, Any], +) -> None: + feature_weights_cfg = dict(local_refine_space.get("feature_weights", {}) or {}) + if feature_weights_cfg: + weights: dict[str, float] = {} + for feature, spec in feature_weights_cfg.items(): + value = float(_local_choice(trial, f"feature_weight__{feature}", spec, 1.0)) + weights[str(feature)] = value + if any(abs(value - 1.0) > 1e-12 for value in weights.values()): + params["feature_weights"] = weights + penalties_cfg = dict(local_refine_space.get("first_feature_use_penalties", {}) or {}) + if penalties_cfg: + penalties: dict[str, float] = {} + for feature, spec in penalties_cfg.items(): + value = float(_local_choice(trial, f"first_use_penalty__{feature}", spec, 0.0)) + penalties[str(feature)] = value + if any(abs(value) > 1e-12 for value in penalties.values()): + params["first_feature_use_penalties"] = penalties + penalties_coeff_spec = local_refine_space.get("penalties_coefficient") + if penalties_coeff_spec is not None: + params["penalties_coefficient"] = float( + _local_choice(trial, "penalties_coefficient", penalties_coeff_spec, 1.0) + ) - def _local_choice(trial: optuna.Trial, name: str, spec: Any, default: Any) -> Any: - if spec is None: - return default - if isinstance(spec, dict): - if spec.get("choices") is not None: - return trial.suggest_categorical(name, list(spec["choices"])) - low = spec.get("low") - high = spec.get("high") - step = spec.get("step") - log = bool(spec.get("log", False)) - if low is None or high is None: - return default - if isinstance(low, int) and isinstance(high, int) and not log: - return trial.suggest_int(name, int(low), int(high), step=int(step or 1)) - return trial.suggest_float( - name, - float(low), - float(high), - step=None if log else (float(step) if step is not None else None), - log=log, - ) - if isinstance(spec, list): - return trial.suggest_categorical(name, list(spec)) - return spec - - def _apply_local_feature_priors(trial: optuna.Trial, params: dict[str, Any]) -> None: - feature_weights_cfg = dict(local_refine_space.get("feature_weights", {}) or {}) - if feature_weights_cfg: - weights: dict[str, float] = {} - for feature, spec in feature_weights_cfg.items(): - value = float(_local_choice(trial, f"feature_weight__{feature}", spec, 1.0)) - weights[str(feature)] = value - if any(abs(value - 1.0) > 1e-12 for value in weights.values()): - params["feature_weights"] = weights - penalties_cfg = dict(local_refine_space.get("first_feature_use_penalties", {}) or {}) - if penalties_cfg: - penalties: dict[str, float] = {} - for feature, spec in penalties_cfg.items(): - value = float(_local_choice(trial, f"first_use_penalty__{feature}", spec, 0.0)) - penalties[str(feature)] = value - if any(abs(value) > 1e-12 for value in penalties.values()): - params["first_feature_use_penalties"] = penalties - penalties_coeff_spec = local_refine_space.get("penalties_coefficient") - if penalties_coeff_spec is not None: - params["penalties_coefficient"] = float( - _local_choice(trial, "penalties_coefficient", penalties_coeff_spec, 1.0) - ) - def _materialize_study_params(sampled_params: dict[str, Any]) -> dict[str, Any]: - params = {**base} - feature_weights: dict[str, float] = {} - penalties: dict[str, float] = {} +def _materialize_study_params( + sampled_params: dict[str, Any], + *, + base: dict[str, Any], + has_monotone_constraints: bool, +) -> dict[str, Any]: + params = {**base} + feature_weights: dict[str, float] = {} + penalties: dict[str, float] = {} + + for key, value in dict(sampled_params or {}).items(): + key_str = str(key) + if key_str.startswith("feature_weight__"): + feature_name = key_str.split("__", 1)[1] + feature_weights[feature_name] = float(value) + continue + if key_str.startswith("first_use_penalty__"): + feature_name = key_str.split("__", 1)[1] + penalties[feature_name] = float(value) + continue + params[key_str] = value + + if feature_weights and any(abs(weight - 1.0) > 1e-12 for weight in feature_weights.values()): + params["feature_weights"] = feature_weights + else: + params.pop("feature_weights", None) + if penalties and any(abs(weight) > 1e-12 for weight in penalties.values()): + params["first_feature_use_penalties"] = penalties + else: + params.pop("first_feature_use_penalties", None) - for key, value in dict(sampled_params or {}).items(): - key_str = str(key) - if key_str.startswith("feature_weight__"): - feature_name = key_str.split("__", 1)[1] - feature_weights[feature_name] = float(value) - continue - if key_str.startswith("first_use_penalty__"): - feature_name = key_str.split("__", 1)[1] - penalties[feature_name] = float(value) - continue - params[key_str] = value + if str(params.get("bootstrap_type", "")).strip() == "Bayesian": + params.pop("subsample", None) + else: + params.pop("bagging_temperature", None) - if feature_weights and any( - abs(weight - 1.0) > 1e-12 for weight in feature_weights.values() - ): - params["feature_weights"] = feature_weights - else: - params.pop("feature_weights", None) - if penalties and any(abs(weight) > 1e-12 for weight in penalties.values()): - params["first_feature_use_penalties"] = penalties - else: - params.pop("first_feature_use_penalties", None) + if str(params.get("grow_policy", "")).strip() == "Lossguide": + params.pop("depth", None) + else: + params.pop("max_leaves", None) + if has_monotone_constraints: + params["grow_policy"] = "SymmetricTree" + params.pop("max_leaves", None) - if str(params.get("bootstrap_type", "")).strip() == "Bayesian": - params.pop("subsample", None) - else: - params.pop("bagging_temperature", None) + if str(params.get("task_type", "")).strip().upper() == "GPU": + params.pop("rsm", None) - if str(params.get("grow_policy", "")).strip() == "Lossguide": - params.pop("depth", None) - else: - params.pop("max_leaves", None) - if has_monotone_constraints: - params["grow_policy"] = "SymmetricTree" - params.pop("max_leaves", None) + return {key: value for key, value in params.items() if value is not None} + + +def _sanitize_enqueued_trial( + raw_params: dict[str, Any], + *, + base: dict[str, Any], + search_space_mode_resolved: str, + has_monotone_constraints: bool, +) -> dict[str, Any]: + """Keep only parameters that are actually sampled by the active Optuna space.""" + allowed = { + "bootstrap_type", + "grow_policy", + "learning_rate", + "l2_leaf_reg", + "min_data_in_leaf", + "random_strength", + "border_count", + "leaf_estimation_iterations", + "rsm", + "depth", + "max_leaves", + "subsample", + "bagging_temperature", + } + if search_space_mode_resolved == "local_refine": + allowed.add("iterations") + params: dict[str, Any] = {} + for key, value in dict(raw_params or {}).items(): + key_str = str(key) + if key_str in allowed or key_str.startswith(("feature_weight__", "first_use_penalty__")): + params[key_str] = value - if str(params.get("task_type", "")).strip().upper() == "GPU": - params.pop("rsm", None) + if has_monotone_constraints: + params["grow_policy"] = "SymmetricTree" + params.pop("max_leaves", None) + if str(params.get("grow_policy", base.get("grow_policy", "SymmetricTree"))) == "Lossguide": + params.pop("depth", None) + else: + params.pop("max_leaves", None) + if str(params.get("bootstrap_type", base.get("bootstrap_type", "MVS"))) == "Bayesian": + params.pop("subsample", None) + else: + params.pop("bagging_temperature", None) + if str(base.get("task_type", "")).strip().upper() == "GPU": + params.pop("rsm", None) + return {key: value for key, value in params.items() if value is not None} + + +def _trial_params_match(left: dict[str, Any], right: dict[str, Any]) -> bool: + if set(left) != set(right): + return False + for key, left_value in left.items(): + right_value = right.get(key) + if right_value is None: + if left_value is not None: + return False + continue + try: + if abs(float(left_value) - float(right_value)) > 1e-12: + return False + except (TypeError, ValueError): + if str(left_value) != str(right_value): + return False + return True - return {key: value for key, value in params.items() if value is not None} - def _sanitize_enqueued_trial(raw_params: dict[str, Any]) -> dict[str, Any]: - """Keep only parameters that are actually sampled by the active Optuna space.""" - allowed = { - "bootstrap_type", - "grow_policy", +def _enqueue_prior_trials( + study: Any, + *, + enqueue_trials: list[dict[str, Any]], + base: dict[str, Any], + search_space_mode_resolved: str, + has_monotone_constraints: bool, +) -> int: + enqueued = 0 + existing = [dict(trial.params) for trial in study.trials] + for raw_params in enqueue_trials: + params = _sanitize_enqueued_trial( + raw_params, + base=base, + search_space_mode_resolved=search_space_mode_resolved, + has_monotone_constraints=has_monotone_constraints, + ) + if not params: + continue + if any(_trial_params_match(params, trial_params) for trial_params in existing): + continue + try: + study.enqueue_trial(params, skip_if_exists=True) + except TypeError: + study.enqueue_trial(params) + existing.append(params) + enqueued += 1 + return enqueued + + +def _local_refine_params( + trial: Any, + *, + base: dict[str, Any], + local_refine_space: dict[str, Any], + feature_order: list[str], + is_gpu: bool, +) -> dict[str, Any]: + params = {**base} + fixed_params = dict(local_refine_space.get("fixed_params", {}) or {}) + params.update(fixed_params) + + params["iterations"] = int( + _local_choice( + trial, + "iterations", + local_refine_space.get("iterations"), + base.get("iterations", 3000), + ) + ) + params["learning_rate"] = float( + _local_choice( + trial, "learning_rate", + local_refine_space.get("learning_rate"), + base.get("learning_rate", 0.03), + ) + ) + params["l2_leaf_reg"] = float( + _local_choice( + trial, "l2_leaf_reg", + local_refine_space.get("l2_leaf_reg"), + base.get("l2_leaf_reg", 3.0), + ) + ) + params["min_data_in_leaf"] = int( + _local_choice( + trial, "min_data_in_leaf", + local_refine_space.get("min_data_in_leaf"), + base.get("min_data_in_leaf", 64), + ) + ) + params["random_strength"] = float( + _local_choice( + trial, "random_strength", + local_refine_space.get("random_strength"), + base.get("random_strength", 1e-6), + ) + ) + params["border_count"] = int( + _local_choice( + trial, "border_count", - "leaf_estimation_iterations", - "rsm", - "depth", - "max_leaves", - "subsample", - "bagging_temperature", - } - if search_space_mode_resolved == "local_refine": - allowed.add("iterations") - params: dict[str, Any] = {} - for key, value in dict(raw_params or {}).items(): - key_str = str(key) - if ( - key_str in allowed - or key_str.startswith(("feature_weight__", "first_use_penalty__")) - ): - params[key_str] = value - - if has_monotone_constraints: - params["grow_policy"] = "SymmetricTree" - params.pop("max_leaves", None) - if str(params.get("grow_policy", base.get("grow_policy", "SymmetricTree"))) == "Lossguide": - params.pop("depth", None) - else: - params.pop("max_leaves", None) - if str(params.get("bootstrap_type", base.get("bootstrap_type", "MVS"))) == "Bayesian": - params.pop("subsample", None) - else: - params.pop("bagging_temperature", None) - if str(base.get("task_type", "")).strip().upper() == "GPU": - params.pop("rsm", None) - return {key: value for key, value in params.items() if value is not None} - - def _trial_params_match(left: dict[str, Any], right: dict[str, Any]) -> bool: - if set(left) != set(right): - return False - for key, left_value in left.items(): - right_value = right.get(key) - if right_value is None: - if left_value is not None: - return False - continue - try: - if abs(float(left_value) - float(right_value)) > 1e-12: - return False - except (TypeError, ValueError): - if str(left_value) != str(right_value): - return False - return True - - def _enqueue_prior_trials(study: optuna.Study) -> int: - enqueued = 0 - existing = [dict(trial.params) for trial in study.trials] - for raw_params in enqueue_trials: - params = _sanitize_enqueued_trial(raw_params) - if not params: - continue - if any(_trial_params_match(params, trial_params) for trial_params in existing): - continue - try: - study.enqueue_trial(params, skip_if_exists=True) - except TypeError: - study.enqueue_trial(params) - existing.append(params) - enqueued += 1 - return enqueued - - def _local_refine_params(trial: optuna.Trial, *, is_gpu: bool) -> dict[str, Any]: - params = {**base} - fixed_params = dict(local_refine_space.get("fixed_params", {}) or {}) - params.update(fixed_params) - - params["iterations"] = int( - _local_choice( - trial, - "iterations", - local_refine_space.get("iterations"), - base.get("iterations", 3000), - ) + local_refine_space.get("border_count"), + base.get("border_count", 128), ) - params["learning_rate"] = float( - _local_choice( - trial, - "learning_rate", - local_refine_space.get("learning_rate"), - base.get("learning_rate", 0.03), - ) + ) + params["leaf_estimation_iterations"] = int( + _local_choice( + trial, + "leaf_estimation_iterations", + local_refine_space.get("leaf_estimation_iterations"), + base.get("leaf_estimation_iterations", 3), ) - params["l2_leaf_reg"] = float( - _local_choice( - trial, - "l2_leaf_reg", - local_refine_space.get("l2_leaf_reg"), - base.get("l2_leaf_reg", 3.0), - ) + ) + params["bootstrap_type"] = _local_choice( + trial, + "bootstrap_type", + local_refine_space.get("bootstrap_type"), + base.get("bootstrap_type", "MVS"), + ) + params["grow_policy"] = _local_choice( + trial, + "grow_policy", + local_refine_space.get("grow_policy"), + base.get("grow_policy", "SymmetricTree"), + ) + + if is_gpu: + params.pop("rsm", None) + else: + params["rsm"] = float( + _local_choice(trial, "rsm", local_refine_space.get("rsm"), base.get("rsm", 1.0)) ) - params["min_data_in_leaf"] = int( - _local_choice( - trial, - "min_data_in_leaf", - local_refine_space.get("min_data_in_leaf"), - base.get("min_data_in_leaf", 64), - ) + if str(params.get("grow_policy", "SymmetricTree")) == "Lossguide": + params.pop("depth", None) + params["max_leaves"] = int( + _local_choice(trial, "max_leaves", local_refine_space.get("max_leaves"), 32) ) - params["random_strength"] = float( - _local_choice( - trial, - "random_strength", - local_refine_space.get("random_strength"), - base.get("random_strength", 1e-6), - ) + else: + params["depth"] = int( + _local_choice(trial, "depth", local_refine_space.get("depth"), base.get("depth", 8)) ) - params["border_count"] = int( + params.pop("max_leaves", None) + if str(params.get("bootstrap_type", "MVS")) == "Bayesian": + params.pop("subsample", None) + bagging_spec = local_refine_space.get("bagging_temperature", {"low": 0.0, "high": 10.0}) + params["bagging_temperature"] = float( _local_choice( trial, - "border_count", - local_refine_space.get("border_count"), - base.get("border_count", 128), + "bagging_temperature", + bagging_spec, + base.get("bagging_temperature", 1.0), ) ) - params["leaf_estimation_iterations"] = int( + else: + params.pop("bagging_temperature", None) + params["subsample"] = float( _local_choice( trial, - "leaf_estimation_iterations", - local_refine_space.get("leaf_estimation_iterations"), - base.get("leaf_estimation_iterations", 3), + "subsample", + local_refine_space.get("subsample"), + base.get("subsample", 0.8), ) ) - params["bootstrap_type"] = _local_choice( - trial, - "bootstrap_type", - local_refine_space.get("bootstrap_type"), - base.get("bootstrap_type", "MVS"), - ) - params["grow_policy"] = _local_choice( - trial, - "grow_policy", - local_refine_space.get("grow_policy"), - base.get("grow_policy", "SymmetricTree"), - ) + _apply_local_feature_priors(trial, params, local_refine_space=local_refine_space) + return _normalize_feature_penalty_params(params, feature_order=feature_order) - if is_gpu: - params.pop("rsm", None) - else: - params["rsm"] = float( - _local_choice(trial, "rsm", local_refine_space.get("rsm"), base.get("rsm", 1.0)) - ) - if str(params.get("grow_policy", "SymmetricTree")) == "Lossguide": - params.pop("depth", None) - params["max_leaves"] = int( - _local_choice(trial, "max_leaves", local_refine_space.get("max_leaves"), 32) - ) - else: - params["depth"] = int( - _local_choice(trial, "depth", local_refine_space.get("depth"), base.get("depth", 8)) - ) - params.pop("max_leaves", None) - if str(params.get("bootstrap_type", "MVS")) == "Bayesian": - params.pop("subsample", None) - bagging_spec = local_refine_space.get("bagging_temperature", {"low": 0.0, "high": 10.0}) - params["bagging_temperature"] = float( - _local_choice( - trial, - "bagging_temperature", - bagging_spec, - base.get("bagging_temperature", 1.0), - ) - ) - else: - params.pop("bagging_temperature", None) - params["subsample"] = float( - _local_choice( - trial, - "subsample", - local_refine_space.get("subsample"), - base.get("subsample", 0.8), - ) - ) - _apply_local_feature_priors(trial, params) - return _normalize_feature_penalty_params(params) - - use_multivariate = bool(multivariate_tpe) - use_group_tpe = bool(group_tpe and use_multivariate) - constraints_func = None - if constraints_policy: - max_brier_delta = constraints_policy.get("max_brier_delta") - max_ece_delta = constraints_policy.get("max_ece_delta") - min_auc_delta = constraints_policy.get("min_auc_delta") - - def constraints_func(frozen_trial: optuna.trial.FrozenTrial) -> list[float]: - attrs = frozen_trial.user_attrs - violations: list[float] = [] - if max_brier_delta is not None: - ceiling = float(incumbent_metrics.get("validation_brier", 0.0)) + float( - max_brier_delta - ) - violations.append(float(attrs.get("validation_brier", float("inf"))) - ceiling) - if max_ece_delta is not None: - ceiling = float(incumbent_metrics.get("validation_ece", 0.0)) + float(max_ece_delta) - violations.append(float(attrs.get("validation_ece", float("inf"))) - ceiling) - if min_auc_delta is not None: - floor = float(incumbent_metrics.get("validation_auc", 0.0)) + float(min_auc_delta) - violations.append(floor - float(attrs.get("validation_auc", float("-inf")))) - return violations - sampler_obj, pruner_obj = _build_optuna_sampler_pruner( - optuna, - sampler=sampler, - pruner=pruner, - n_startup_trials=n_startup_trials, - multivariate_tpe=use_multivariate, - group_tpe=use_group_tpe, - constant_liar=constant_liar, - pruner_n_startup_trials=pruner_n_startup_trials, - pruner_n_warmup_steps=pruner_n_warmup_steps, - constraints_func=constraints_func, +def _global_search_params( + trial: Any, + *, + base: dict[str, Any], + has_monotone_constraints: bool, +) -> dict[str, Any]: + bootstrap_type = trial.suggest_categorical("bootstrap_type", ["Bayesian", "Bernoulli", "MVS"]) + grow_policy_choices = ( + ["SymmetricTree"] + if has_monotone_constraints + else ["SymmetricTree", "Depthwise", "Lossguide"] ) + grow_policy = trial.suggest_categorical("grow_policy", grow_policy_choices) + params = { + **base, + "learning_rate": trial.suggest_float("learning_rate", 0.005, 0.20, log=True), + "l2_leaf_reg": trial.suggest_float("l2_leaf_reg", 0.5, 100.0, log=True), + "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 20, 500), + "random_strength": trial.suggest_float("random_strength", 1e-9, 10.0, log=True), + "border_count": trial.suggest_int("border_count", 64, 254), + "bootstrap_type": bootstrap_type, + "grow_policy": grow_policy, + "leaf_estimation_iterations": trial.suggest_int("leaf_estimation_iterations", 1, 10), + "random_seed": int(base.get("random_seed", 42)), + } + if str(base.get("task_type", "")).strip().upper() == "GPU": + params.pop("rsm", None) + else: + params["rsm"] = trial.suggest_float("rsm", 0.5, 1.0) + if grow_policy == "Lossguide": + params["max_leaves"] = trial.suggest_int("max_leaves", 16, 64) + params.pop("depth", None) + else: + params["depth"] = trial.suggest_int("depth", 4, 10) + if bootstrap_type == "Bayesian": + params.pop("subsample", None) + params["bagging_temperature"] = trial.suggest_float("bagging_temperature", 0.0, 10.0) + else: + params.pop("bagging_temperature", None) + params["subsample"] = trial.suggest_float("subsample", 0.5, 0.95) + return params - train_pool = Pool(X_train, y_train, cat_features=cat_features, weight=sample_weight) - val_pool = Pool(X_val, y_val, cat_features=cat_features, weight=eval_sample_weight) - if constraints_policy: - incumbent_model = CatBoostClassifier(**base) - incumbent_model.fit(train_pool, eval_set=val_pool, use_best_model=True) - incumbent_y_val_prob = incumbent_model.predict_proba(X_val)[:, 1] - incumbent_metrics = { - "validation_auc": float(roc_auc_score(y_val, incumbent_y_val_prob)), - "validation_brier": float(brier_score_loss(y_val, incumbent_y_val_prob)), - "validation_ece": float(expected_calibration_error(y_val, incumbent_y_val_prob)), - } - - def objective(trial: optuna.Trial) -> float: - is_gpu = str(base.get("task_type", "")).strip().upper() == "GPU" - if search_space_mode_resolved == "local_refine": - params = _local_refine_params(trial, is_gpu=is_gpu) - else: - bootstrap_type = trial.suggest_categorical( - "bootstrap_type", ["Bayesian", "Bernoulli", "MVS"] - ) - grow_policy_choices = ( - ["SymmetricTree"] - if has_monotone_constraints - else ["SymmetricTree", "Depthwise", "Lossguide"] + +def _constraints_violations( + frozen_trial: Any, + *, + incumbent_metrics: dict[str, float], + constraints_policy: dict[str, Any], +) -> list[float]: + attrs = frozen_trial.user_attrs + violations: list[float] = [] + max_brier_delta = constraints_policy.get("max_brier_delta") + max_ece_delta = constraints_policy.get("max_ece_delta") + min_auc_delta = constraints_policy.get("min_auc_delta") + if max_brier_delta is not None: + ceiling = float(incumbent_metrics.get("validation_brier", 0.0)) + float(max_brier_delta) + violations.append(float(attrs.get("validation_brier", float("inf"))) - ceiling) + if max_ece_delta is not None: + ceiling = float(incumbent_metrics.get("validation_ece", 0.0)) + float(max_ece_delta) + violations.append(float(attrs.get("validation_ece", float("inf"))) - ceiling) + if min_auc_delta is not None: + floor = float(incumbent_metrics.get("validation_auc", 0.0)) + float(min_auc_delta) + violations.append(floor - float(attrs.get("validation_auc", float("-inf")))) + return violations + + +def _incumbent_validation_metrics( + *, + base: dict[str, Any], + train_pool: Pool, + val_pool: Pool, + X_val: pd.DataFrame, + y_val: pd.Series, +) -> dict[str, float]: + incumbent_model = CatBoostClassifier(**base) + incumbent_model.fit(train_pool, eval_set=val_pool, use_best_model=True) + incumbent_y_val_prob = incumbent_model.predict_proba(X_val)[:, 1] + y_val_array = np.asarray(y_val, dtype=int) + return { + "validation_auc": float(roc_auc_score(y_val, incumbent_y_val_prob)), + "validation_brier": float(brier_score_loss(y_val, incumbent_y_val_prob)), + "validation_ece": float(expected_calibration_error(y_val_array, incumbent_y_val_prob)), + } + + +def _catboost_pruning_callbacks( + trial: Any, *, use_pruning_callback: bool +) -> tuple[Any | None, list[Any]]: + if not use_pruning_callback: + return None, [] + try: + from optuna_integration.catboost import CatBoostPruningCallback + + pruning_callback = CatBoostPruningCallback(trial, "AUC") + return pruning_callback, [pruning_callback] + except Exception as exc: # pragma: no cover - optional integration path + if trial.number == 0: + logger.warning( + "CatBoost pruning callback unavailable; disabling pruning callback: {}", exc ) - grow_policy = trial.suggest_categorical("grow_policy", grow_policy_choices) - params = { - **base, - "learning_rate": trial.suggest_float("learning_rate", 0.005, 0.20, log=True), - "l2_leaf_reg": trial.suggest_float("l2_leaf_reg", 0.5, 100.0, log=True), - "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 20, 500), - "random_strength": trial.suggest_float("random_strength", 1e-9, 10.0, log=True), - "border_count": trial.suggest_int("border_count", 64, 254), - "bootstrap_type": bootstrap_type, - "grow_policy": grow_policy, - "leaf_estimation_iterations": trial.suggest_int( - "leaf_estimation_iterations", 1, 10 - ), - "random_seed": int(base.get("random_seed", 42)), - } - if is_gpu: - params.pop("rsm", None) - else: - params["rsm"] = trial.suggest_float("rsm", 0.5, 1.0) - if grow_policy == "Lossguide": - params["max_leaves"] = trial.suggest_int("max_leaves", 16, 64) - params.pop("depth", None) - else: - params["depth"] = trial.suggest_int("depth", 4, 10) - if bootstrap_type == "Bayesian": - params.pop("subsample", None) - params["bagging_temperature"] = trial.suggest_float( - "bagging_temperature", 0.0, 10.0 - ) - else: - params.pop("bagging_temperature", None) - params["subsample"] = trial.suggest_float("subsample", 0.5, 0.95) - - params = _normalize_feature_penalty_params(params) - model = CatBoostClassifier(**params) - pruning_callback = None - callbacks: list[Any] = [] - if use_pruning_callback: - try: - from optuna_integration.catboost import CatBoostPruningCallback - - pruning_callback = CatBoostPruningCallback(trial, "AUC") - callbacks = [pruning_callback] - except Exception as exc: # pragma: no cover - optional integration path - if trial.number == 0: - logger.warning( - "CatBoost pruning callback unavailable; disabling pruning callback: {}", exc - ) - pruning_callback = None - callbacks = [] + return None, [] - model.fit( - train_pool, - eval_set=val_pool, - use_best_model=True, - callbacks=callbacks or None, + +def _catboost_objective( + trial: Any, + *, + base: dict[str, Any], + search_space_mode_resolved: str, + local_refine_space: dict[str, Any], + feature_order: list[str], + has_monotone_constraints: bool, + train_pool: Pool, + val_pool: Pool, + X_val: pd.DataFrame, + y_val: pd.Series, + use_pruning_callback: bool, +) -> float: + is_gpu = str(base.get("task_type", "")).strip().upper() == "GPU" + if search_space_mode_resolved == "local_refine": + params = _local_refine_params( + trial, + base=base, + local_refine_space=local_refine_space, + feature_order=feature_order, + is_gpu=is_gpu, + ) + else: + params = _global_search_params( + trial, + base=base, + has_monotone_constraints=has_monotone_constraints, ) - if pruning_callback is not None: - pruning_callback.check_pruned() + params = _normalize_feature_penalty_params(params, feature_order=feature_order) + model = CatBoostClassifier(**params) + pruning_callback, callbacks = _catboost_pruning_callbacks( + trial, + use_pruning_callback=use_pruning_callback, + ) + model.fit( + train_pool, + eval_set=val_pool, + use_best_model=True, + callbacks=callbacks or None, + ) - val_auc = model.get_best_score().get("validation", {}).get("AUC") - if val_auc is None: - y_val_prob = model.predict_proba(X_val)[:, 1] - val_auc = roc_auc_score(y_val, y_val_prob) - else: - y_val_prob = model.predict_proba(X_val)[:, 1] - val_brier = float(brier_score_loss(y_val, y_val_prob)) - val_ece = float(expected_calibration_error(y_val, y_val_prob)) + if pruning_callback is not None: + pruning_callback.check_pruned() + + val_auc = model.get_best_score().get("validation", {}).get("AUC") + y_val_prob = model.predict_proba(X_val)[:, 1] + if val_auc is None: + val_auc = roc_auc_score(y_val, y_val_prob) + y_val_array = np.asarray(y_val, dtype=int) + trial.set_user_attr("best_iteration", int(model.get_best_iteration())) + trial.set_user_attr("validation_auc", float(val_auc)) + trial.set_user_attr("validation_brier", float(brier_score_loss(y_val_array, y_val_prob))) + trial.set_user_attr( + "validation_ece", + float(expected_calibration_error(y_val_array, y_val_prob)), + ) + return float(val_auc) - trial.set_user_attr("best_iteration", int(model.get_best_iteration())) - trial.set_user_attr("validation_auc", float(val_auc)) - trial.set_user_attr("validation_brier", val_brier) - trial.set_user_attr("validation_ece", val_ece) - return float(val_auc) +def _create_optuna_study( + optuna_module: Any, + *, + sampler_obj: Any, + pruner_obj: Any, + study_storage: str | None, + study_name: str | None, + load_if_exists: bool, + search_space_version: str, + storage_heartbeat_interval: int, + storage_grace_period: int, + sqlite_timeout_seconds: int, + retry_failed_trials: int, +) -> tuple[Any, Any | None]: create_study_kwargs: dict[str, Any] = { "direction": "maximize", "sampler": sampler_obj, "pruner": pruner_obj, } retry_callback = None - if study_storage: - storage_obj: Any = study_storage - hb_interval = max(0, int(storage_heartbeat_interval)) - hb_grace = max(0, int(storage_grace_period)) - storage_text = str(study_storage) - if _is_journal_storage_url(storage_text): - journal_path = _journal_path_from_storage_url(storage_text) - Path(journal_path).parent.mkdir(parents=True, exist_ok=True) - from src.utils.optuna_storage import _make_journal_storage - - storage_obj = _make_journal_storage( - Path(journal_path), - study_name=resolve_optuna_study_name( - study_name, - search_space_version=search_space_version, - ), - ) - else: - # For long-running trials on SQLite, use a longer connection timeout and - # heartbeat to recover stale RUNNING trials after crashes/restarts. - if storage_text.startswith(("sqlite:///", "sqlite+pysqlite:///")): - engine_kwargs = {"connect_args": {"timeout": max(1, int(sqlite_timeout_seconds))}} - else: - engine_kwargs = None - if hb_interval > 0 or hb_grace > 0: - try: - heartbeat_cb = None - failed_cb = None - if int(retry_failed_trials) > 0: - retry_factory = getattr( - optuna.storages, - "RetryHeartbeatStaleTrialCallback", - None, - ) - if retry_factory is not None: - heartbeat_cb = retry_factory(max_retry=int(retry_failed_trials)) - retry_callback = heartbeat_cb - else: # pragma: no cover - compatibility with older Optuna. - failed_cb = optuna.storages.RetryFailedTrialCallback( - max_retry=int(retry_failed_trials) - ) - retry_callback = failed_cb - storage_kwargs: dict[str, Any] = { - "url": storage_text, - "engine_kwargs": engine_kwargs, - "heartbeat_interval": hb_interval or None, - "grace_period": hb_grace or None, - } - if heartbeat_cb is not None: - storage_kwargs["heartbeat_stale_trial_callback"] = heartbeat_cb - elif failed_cb is not None: - storage_kwargs["failed_trial_callback"] = failed_cb - storage_obj = optuna.storages.RDBStorage(**storage_kwargs) - except Exception as exc: - logger.warning( - "Optuna RDBStorage heartbeat/retry setup failed; falling back to storage " - "URL. reason={}", - exc, - ) - create_study_kwargs["storage"] = storage_obj - create_study_kwargs["study_name"] = resolve_optuna_study_name( - study_name, - search_space_version=search_space_version, + if not study_storage: + return optuna_module.create_study(**create_study_kwargs), retry_callback + + storage_obj: Any = study_storage + hb_interval = max(0, int(storage_heartbeat_interval)) + hb_grace = max(0, int(storage_grace_period)) + storage_text = str(study_storage) + if _is_journal_storage_url(storage_text): + journal_path = _journal_path_from_storage_url(storage_text) + Path(journal_path).parent.mkdir(parents=True, exist_ok=True) + from src.utils.optuna_storage import _make_journal_storage + + storage_obj = _make_journal_storage( + Path(journal_path), + study_name=resolve_optuna_study_name( + study_name, + search_space_version=search_space_version, + ), + ) + else: + engine_kwargs = ( + {"connect_args": {"timeout": max(1, int(sqlite_timeout_seconds))}} + if storage_text.startswith(("sqlite:///", "sqlite+pysqlite:///")) + else None ) - create_study_kwargs["load_if_exists"] = bool(load_if_exists) + if hb_interval > 0 or hb_grace > 0: + try: + heartbeat_cb = None + failed_cb = None + if int(retry_failed_trials) > 0: + retry_factory = getattr( + optuna_module.storages, + "RetryHeartbeatStaleTrialCallback", + None, + ) + if retry_factory is not None: + heartbeat_cb = retry_factory(max_retry=int(retry_failed_trials)) + retry_callback = heartbeat_cb + else: # pragma: no cover - compatibility with older Optuna. + failed_cb = optuna_module.storages.RetryFailedTrialCallback( + max_retry=int(retry_failed_trials) + ) + retry_callback = failed_cb + storage_kwargs: dict[str, Any] = { + "url": storage_text, + "engine_kwargs": engine_kwargs, + "heartbeat_interval": hb_interval or None, + "grace_period": hb_grace or None, + } + if heartbeat_cb is not None: + storage_kwargs["heartbeat_stale_trial_callback"] = heartbeat_cb + elif failed_cb is not None: + storage_kwargs["failed_trial_callback"] = failed_cb + storage_obj = optuna_module.storages.RDBStorage(**storage_kwargs) + except Exception as exc: + logger.warning( + "Optuna RDBStorage heartbeat/retry setup failed; falling back to storage " + "URL. reason={}", + exc, + ) + + create_study_kwargs["storage"] = storage_obj + create_study_kwargs["study_name"] = resolve_optuna_study_name( + study_name, + search_space_version=search_space_version, + ) + create_study_kwargs["load_if_exists"] = bool(load_if_exists) + return optuna_module.create_study(**create_study_kwargs), retry_callback + + +def _local_refine_base_trial_params(base: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in { + "iterations": int(base.get("iterations", 3000)), + "learning_rate": float(base.get("learning_rate", 0.03)), + "l2_leaf_reg": float(base.get("l2_leaf_reg", 3.0)), + "min_data_in_leaf": int(base.get("min_data_in_leaf", 64)), + "random_strength": float(base.get("random_strength", 1e-6)), + "border_count": int(base.get("border_count", 128)), + "leaf_estimation_iterations": int(base.get("leaf_estimation_iterations", 3)), + "bootstrap_type": str(base.get("bootstrap_type", "MVS")), + "grow_policy": str(base.get("grow_policy", "SymmetricTree")), + "rsm": None + if str(base.get("task_type", "")).strip().upper() == "GPU" + else float(base.get("rsm", 1.0)), + "depth": None + if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" + else int(base.get("depth", 8)), + "max_leaves": int(base.get("max_leaves", 32)) + if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" + else None, + "subsample": None + if str(base.get("bootstrap_type", "MVS")) == "Bayesian" + else float(base.get("subsample", 0.8)), + "bagging_temperature": float(base.get("bagging_temperature", 1.0)) + if str(base.get("bootstrap_type", "MVS")) == "Bayesian" + else None, + }.items() + if value is not None + } + - study = optuna.create_study(**create_study_kwargs) - n_enqueued_prior_trials = _enqueue_prior_trials(study) - if search_space_mode_resolved == "local_refine" and bool( +def _enqueue_local_refine_base_trial( + study: Any, + *, + base: dict[str, Any], + local_refine_space: dict[str, Any], + search_space_mode_resolved: str, +) -> None: + if search_space_mode_resolved != "local_refine" or not bool( local_refine_space.get("enqueue_base_trial", True) ): - try: - study.enqueue_trial( - { - key: value - for key, value in { - "iterations": int(base.get("iterations", 3000)), - "learning_rate": float(base.get("learning_rate", 0.03)), - "l2_leaf_reg": float(base.get("l2_leaf_reg", 3.0)), - "min_data_in_leaf": int(base.get("min_data_in_leaf", 64)), - "random_strength": float(base.get("random_strength", 1e-6)), - "border_count": int(base.get("border_count", 128)), - "leaf_estimation_iterations": int( - base.get("leaf_estimation_iterations", 3) - ), - "bootstrap_type": str(base.get("bootstrap_type", "MVS")), - "grow_policy": str(base.get("grow_policy", "SymmetricTree")), - "rsm": None - if str(base.get("task_type", "")).strip().upper() == "GPU" - else float(base.get("rsm", 1.0)), - "depth": None - if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" - else int(base.get("depth", 8)), - "max_leaves": int(base.get("max_leaves", 32)) - if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" - else None, - "subsample": None - if str(base.get("bootstrap_type", "MVS")) == "Bayesian" - else float(base.get("subsample", 0.8)), - "bagging_temperature": float(base.get("bagging_temperature", 1.0)) - if str(base.get("bootstrap_type", "MVS")) == "Bayesian" - else None, - }.items() - if value is not None - } - ) - except Exception as exc: - logger.warning("Optuna enqueue_trial for local_refine skipped: {}", exc) - if retry_callback is not None and hasattr(optuna.storages, "fail_stale_trials"): - try: - optuna.storages.fail_stale_trials(study) - except Exception as exc: - logger.warning("Optuna stale-trial recovery skipped: {}", exc) + return + try: + study.enqueue_trial(_local_refine_base_trial_params(base)) + except Exception as exc: + logger.warning("Optuna enqueue_trial for local_refine skipped: {}", exc) + + +def _complete_trial_value(trial: Any) -> float: + value = trial.value + if value is None: + raise ValueError("COMPLETE Optuna trial has no objective value.") + return float(value) + + +def _complete_trials(study: Any, optuna_module: Any) -> list[Any]: + return [ + trial + for trial in study.trials + if trial.state == optuna_module.trial.TrialState.COMPLETE and trial.value is not None + ] + + +def _selected_optuna_trial(study: Any, optuna_module: Any) -> tuple[Any, float, bool]: + complete_trials = _complete_trials(study, optuna_module) + if not complete_trials: + raise ValueError("Optuna study has no COMPLETE trials available for model selection.") + try: + selected_trial = study.best_trial + constrained_best_trial = True + except ValueError as exc: + selected_trial = max(complete_trials, key=_complete_trial_value) + constrained_best_trial = False + logger.warning( + "Optuna study has no feasible trial under constraints; using best COMPLETE trial " + "by validation AUC as fallback. reason={}", + exc, + ) + return selected_trial, _complete_trial_value(selected_trial), constrained_best_trial + + +def _run_or_reuse_study( + study: Any, + *, + objective: Any, + n_trials: int, + timeout_minutes: int, + gc_after_trial: bool, + n_jobs: int, +) -> None: timeout = None if timeout_minutes <= 0 else int(timeout_minutes * 60) requested_trials = int(n_trials) if requested_trials > 0: @@ -739,7 +841,9 @@ def objective(trial: optuna.Trial) -> float: ) else: complete_trials = [ - t for t in study.trials if t.state.name == "COMPLETE" and t.value is not None + trial + for trial in study.trials + if trial.state.name == "COMPLETE" and trial.value is not None ] if not complete_trials: raise ValueError( @@ -753,63 +857,252 @@ def objective(trial: optuna.Trial) -> float: ) gc.collect() - complete_trials = [ - trial - for trial in study.trials - if trial.state == optuna.trial.TrialState.COMPLETE and trial.value is not None - ] - if not complete_trials: - raise ValueError("Optuna study has no COMPLETE trials available for model selection.") - constrained_best_trial = True - try: - selected_trial = study.best_trial - except ValueError as exc: - selected_trial = max(complete_trials, key=lambda trial: float(trial.value)) - constrained_best_trial = False - logger.warning( - "Optuna study has no feasible trial under constraints; using best COMPLETE trial " - "by validation AUC as fallback. reason={}", - exc, - ) - selected_params = dict(selected_trial.params) - selected_value = float(selected_trial.value) - best_params = _materialize_study_params(selected_params) +def _fit_selected_catboost_model( + *, + selected_trial: Any, + base: dict[str, Any], + has_monotone_constraints: bool, + feature_order: list[str], + train_pool: Pool, + val_pool: Pool, + X_train: pd.DataFrame, + y_train: pd.Series, + X_val: pd.DataFrame, + y_val: pd.Series, + cat_features: list[str], + sample_weight: np.ndarray | None, + eval_sample_weight: np.ndarray | None, + refit_full_train: bool, +) -> _SelectedModelFit: + selected_params = dict(selected_trial.params) + best_params = _materialize_study_params( + selected_params, + base=base, + has_monotone_constraints=has_monotone_constraints, + ) best_params["verbose"] = 100 - catboost_best_params = _normalize_feature_penalty_params(best_params) + catboost_best_params = _normalize_feature_penalty_params( + best_params, + feature_order=feature_order, + ) selection_model = CatBoostClassifier(**catboost_best_params) selection_model.fit(train_pool, eval_set=val_pool, use_best_model=True) y_val_prob = selection_model.predict_proba(X_val)[:, 1] - val_auc = roc_auc_score(y_val, y_val_prob) + val_auc = float(roc_auc_score(y_val, y_val_prob)) best_iteration = int(selection_model.get_best_iteration()) - if refit_full_train: - full_X = pd.concat([X_train, X_val], axis=0).reset_index(drop=True) - full_y = pd.concat([y_train, y_val], axis=0).reset_index(drop=True) - full_weight = None - if sample_weight is not None and eval_sample_weight is not None: - full_weight = np.concatenate( - [ - np.asarray(sample_weight, dtype=float), - np.asarray(eval_sample_weight, dtype=float), - ] + if not refit_full_train: + return _SelectedModelFit( + model=selection_model, + selected_params=selected_params, + resolved_params=best_params, + validation_auc=val_auc, + best_iteration=best_iteration, + ) + + full_X = pd.concat([X_train, X_val], axis=0).reset_index(drop=True) + full_y = pd.concat([y_train, y_val], axis=0).reset_index(drop=True) + full_weight = None + if sample_weight is not None and eval_sample_weight is not None: + full_weight = np.concatenate( + [ + np.asarray(sample_weight, dtype=float), + np.asarray(eval_sample_weight, dtype=float), + ] + ) + full_pool = Pool(full_X, full_y, cat_features=cat_features, weight=full_weight) + refit_params = { + key: value for key, value in catboost_best_params.items() if key != "early_stopping_rounds" + } + if best_iteration > 0: + refit_params["iterations"] = best_iteration + 1 + best_model = CatBoostClassifier(**refit_params) + best_model.fit(full_pool) + return _SelectedModelFit( + model=best_model, + selected_params=selected_params, + resolved_params=best_params, + validation_auc=val_auc, + best_iteration=best_iteration, + ) + + +def train_catboost_tuned_optuna( + X_train: pd.DataFrame, + y_train: pd.Series, + X_val: pd.DataFrame, + y_val: pd.Series, + X_test: pd.DataFrame | None = None, + y_test: pd.Series | None = None, + *, + cat_features: list[str] | None = None, + base_params: dict[str, Any] | None = None, + n_trials: int = 100, + sampler: str = "tpe", + pruner: str = "median", + timeout_minutes: int = 0, + n_startup_trials: int = 40, + multivariate_tpe: bool = True, + group_tpe: bool = True, + warn_independent_sampling: bool = True, + constant_liar: bool = False, + pruner_n_startup_trials: int = 20, + pruner_n_warmup_steps: int = 50, + use_pruning_callback: bool = True, + study_storage: str | None = None, + study_name: str | None = None, + load_if_exists: bool = True, + refit_full_train: bool = True, + gc_after_trial: bool = True, + storage_heartbeat_interval: int = 0, + storage_grace_period: int = 0, + sqlite_timeout_seconds: int = 60, + retry_failed_trials: int = 0, + n_jobs: int = 1, + sample_weight: np.ndarray | None = None, + eval_sample_weight: np.ndarray | None = None, + search_space_mode: str = "global", + local_refine_space: dict[str, Any] | None = None, + constraints_policy: dict[str, Any] | None = None, + search_space_version: str = SEARCH_SPACE_VERSION, + enqueue_trials: list[dict[str, Any]] | None = None, +) -> tuple[CatBoostClassifier, dict[str, Any]]: + """Tune CatBoost with Optuna and return best fitted model and metadata.""" + import optuna + + if cat_features is None: + cat_features = [c for c in CATEGORICAL_FEATURES if c in X_train.columns] + + base = _catboost_base_params(base_params) + base["verbose"] = 0 + has_monotone_constraints = bool(str(base.get("monotone_constraints", "")).strip()) + if has_monotone_constraints: + base["grow_policy"] = "SymmetricTree" + search_space_mode_resolved = str(search_space_mode or "global").strip().lower() or "global" + local_refine_space = dict(local_refine_space or {}) + if has_monotone_constraints: + local_refine_space["grow_policy"] = ["SymmetricTree"] + constraints_policy = dict(constraints_policy or {}) + enqueue_trials = list(enqueue_trials or []) + feature_order = [str(col) for col in X_train.columns] + + train_pool = Pool(X_train, y_train, cat_features=cat_features, weight=sample_weight) + val_pool = Pool(X_val, y_val, cat_features=cat_features, weight=eval_sample_weight) + incumbent_metrics: dict[str, float] = {} + if constraints_policy: + incumbent_metrics.update( + _incumbent_validation_metrics( + base=base, + train_pool=train_pool, + val_pool=val_pool, + X_val=X_val, + y_val=y_val, ) - full_pool = Pool(full_X, full_y, cat_features=cat_features, weight=full_weight) - refit_params = { - k: v for k, v in catboost_best_params.items() if k != "early_stopping_rounds" - } - if best_iteration > 0: - refit_params["iterations"] = best_iteration + 1 - best_model = CatBoostClassifier(**refit_params) - best_model.fit(full_pool) - else: - best_model = selection_model + ) + + use_multivariate = bool(multivariate_tpe) + use_group_tpe = bool(group_tpe and use_multivariate) + constraints_func = ( + partial( + _constraints_violations, + incumbent_metrics=incumbent_metrics, + constraints_policy=constraints_policy, + ) + if constraints_policy + else None + ) + + sampler_obj, pruner_obj = _build_optuna_sampler_pruner( + optuna, + sampler=sampler, + pruner=pruner, + n_startup_trials=n_startup_trials, + multivariate_tpe=use_multivariate, + group_tpe=use_group_tpe, + constant_liar=constant_liar, + pruner_n_startup_trials=pruner_n_startup_trials, + pruner_n_warmup_steps=pruner_n_warmup_steps, + constraints_func=constraints_func, + ) + + objective = partial( + _catboost_objective, + base=base, + search_space_mode_resolved=search_space_mode_resolved, + local_refine_space=local_refine_space, + feature_order=feature_order, + has_monotone_constraints=has_monotone_constraints, + train_pool=train_pool, + val_pool=val_pool, + X_val=X_val, + y_val=y_val, + use_pruning_callback=use_pruning_callback, + ) + + study, retry_callback = _create_optuna_study( + optuna, + sampler_obj=sampler_obj, + pruner_obj=pruner_obj, + study_storage=study_storage, + study_name=study_name, + load_if_exists=load_if_exists, + search_space_version=search_space_version, + storage_heartbeat_interval=storage_heartbeat_interval, + storage_grace_period=storage_grace_period, + sqlite_timeout_seconds=sqlite_timeout_seconds, + retry_failed_trials=retry_failed_trials, + ) + n_enqueued_prior_trials = _enqueue_prior_trials( + study, + enqueue_trials=enqueue_trials, + base=base, + search_space_mode_resolved=search_space_mode_resolved, + has_monotone_constraints=has_monotone_constraints, + ) + _enqueue_local_refine_base_trial( + study, + base=base, + local_refine_space=local_refine_space, + search_space_mode_resolved=search_space_mode_resolved, + ) + if retry_callback is not None and hasattr(optuna.storages, "fail_stale_trials"): + try: + optuna.storages.fail_stale_trials(study) + except Exception as exc: + logger.warning("Optuna stale-trial recovery skipped: {}", exc) + _run_or_reuse_study( + study, + objective=objective, + n_trials=n_trials, + timeout_minutes=timeout_minutes, + gc_after_trial=gc_after_trial, + n_jobs=n_jobs, + ) + + selected_trial, selected_value, constrained_best_trial = _selected_optuna_trial(study, optuna) + selected_fit = _fit_selected_catboost_model( + selected_trial=selected_trial, + base=base, + has_monotone_constraints=has_monotone_constraints, + feature_order=feature_order, + train_pool=train_pool, + val_pool=val_pool, + X_train=X_train, + y_train=y_train, + X_val=X_val, + y_val=y_val, + cat_features=cat_features, + sample_weight=sample_weight, + eval_sample_weight=eval_sample_weight, + refit_full_train=refit_full_train, + ) metrics: dict[str, Any] = { - "validation_auc": float(val_auc), - "best_iteration": best_iteration, - "best_params": selected_params, - "best_params_resolved": best_params, + "validation_auc": float(selected_fit.validation_auc), + "best_iteration": selected_fit.best_iteration, + "best_params": selected_fit.selected_params, + "best_params_resolved": selected_fit.resolved_params, "hpo_trials_executed": len(study.trials), "hpo_best_validation_auc": selected_value, "hpo_selected_trial_number": int(selected_trial.number), @@ -822,15 +1115,15 @@ def objective(trial: optuna.Trial) -> float: "enqueued_prior_trials": n_enqueued_prior_trials, } if X_test is not None and y_test is not None: - y_test_prob = best_model.predict_proba(X_test)[:, 1] + y_test_prob = selected_fit.model.predict_proba(X_test)[:, 1] metrics["auc_roc"] = float(roc_auc_score(y_test, y_test_prob)) logger.info( "CatBoost tuned — val_AUC: " - f"{val_auc:.4f}, best_trial_val_AUC: {selected_value:.4f}, " + f"{selected_fit.validation_auc:.4f}, best_trial_val_AUC: {selected_value:.4f}, " f"trials={len(study.trials)}, multivariate_tpe={use_multivariate}, group_tpe={use_group_tpe}" ) - return best_model, metrics + return selected_fit.model, metrics def export_hpo_visualizations( diff --git a/src/optimization/certificate_semantics.py b/src/optimization/certificate_semantics.py new file mode 100644 index 0000000..345d9df --- /dev/null +++ b/src/optimization/certificate_semantics.py @@ -0,0 +1,260 @@ +"""Shared semantics for the paper-facing funded-set decision certificate.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +IJDS_DECLARED_ALPHA_GRID: tuple[float, ...] = ( + 0.01, + 0.03, + 0.05, + 0.07, + 0.10, + 0.12, + 0.15, + 0.20, +) +IJDS_DECLARED_ALPHA_GRID_CSV = ",".join(f"{alpha:.2f}" for alpha in IJDS_DECLARED_ALPHA_GRID) + + +@dataclass(frozen=True) +class FundedCertificateMetrics: + """Exact funded-set metrics and their policy-aware upper bounds. + + ``endpoint_budget`` and ``markov_loss_threshold`` are exact for the supplied + funded weights. ``endpoint_budget_upper`` and ``markov_loss_cap`` additionally + use the declared effective-PD constraint and its optional solver slack. + """ + + alpha: float + risk_tolerance: float + n_funded: int + weighted_outcome: float + weighted_miscoverage: float + weighted_coverage: float + empirical_coverage_funded: float + weighted_pd_point: float + weighted_pd_effective: float + endpoint_budget: float + gamma_cp: float + gamma_internalized: float + gamma_residual: float + effective_constraint_slack: float + effective_constraint_excess: float + realized_risk_tolerance_excess: float + sqrt_alpha: float + endpoint_budget_upper: float + markov_loss_threshold: float + markov_loss_cap: float + + +def _one_dimensional_float_array(values: np.ndarray, *, name: str) -> np.ndarray: + array = np.asarray(values, dtype=float) + if array.ndim != 1: + raise ValueError(f"{name} must be one-dimensional, got shape={array.shape}") + if not np.all(np.isfinite(array)): + raise ValueError(f"{name} contains non-finite values.") + return array + + +def _validate_probability_array(values: np.ndarray, *, name: str) -> None: + tolerance = 1e-10 + if np.any(values < -tolerance) or np.any(values > 1.0 + tolerance): + raise ValueError(f"{name} must stay on the [0, 1] probability scale.") + + +def compute_funded_certificate_metrics( + weights: np.ndarray, + outcomes: np.ndarray, + pd_point: np.ndarray, + pd_high: np.ndarray, + pd_effective: np.ndarray, + *, + alpha: float, + risk_tolerance: float, + pd_cap_slack: float = 0.0, + funded_tolerance: float = 1e-8, +) -> FundedCertificateMetrics: + """Compute the exact and policy-aware funded-set certificate. + + The effective PD vector may come from any policy family. This is important + for capped and tail-focused policies, where the linear-blend shortcut + ``(1 - gamma) * Gamma_CP`` is not generally the residual endpoint premium. + """ + arrays = { + "weights": _one_dimensional_float_array(weights, name="weights"), + "outcomes": _one_dimensional_float_array(outcomes, name="outcomes"), + "pd_point": _one_dimensional_float_array(pd_point, name="pd_point"), + "pd_high": _one_dimensional_float_array(pd_high, name="pd_high"), + "pd_effective": _one_dimensional_float_array(pd_effective, name="pd_effective"), + } + lengths = {len(values) for values in arrays.values()} + if len(lengths) != 1: + shapes = {name: values.shape for name, values in arrays.items()} + raise ValueError(f"Certificate arrays must have the same length: {shapes}") + + weights_arr = arrays["weights"] + outcomes_arr = arrays["outcomes"] + point_arr = arrays["pd_point"] + high_arr = arrays["pd_high"] + effective_arr = arrays["pd_effective"] + if np.any(weights_arr < 0.0): + raise ValueError("weights must be nonnegative.") + weight_sum = float(weights_arr.sum()) + if not np.isclose(weight_sum, 1.0, rtol=0.0, atol=1e-8): + raise ValueError(f"weights must sum to one, got {weight_sum:.12g}") + + for name, values in ( + ("outcomes", outcomes_arr), + ("pd_point", point_arr), + ("pd_high", high_arr), + ("pd_effective", effective_arr), + ): + _validate_probability_array(values, name=name) + order_tolerance = 1e-10 + if np.any(high_arr + order_tolerance < point_arr): + raise ValueError("pd_high must be at least pd_point for every row.") + if np.any(effective_arr + order_tolerance < point_arr) or np.any( + effective_arr > high_arr + order_tolerance + ): + raise ValueError("pd_effective must lie between pd_point and pd_high.") + + alpha_value = float(alpha) + if not 0.0 < alpha_value < 1.0: + raise ValueError(f"alpha must lie in (0, 1), got {alpha_value}") + tolerance_value = float(risk_tolerance) + if not 0.0 <= tolerance_value <= 1.0: + raise ValueError(f"risk_tolerance must lie in [0, 1], got {tolerance_value}") + cap_slack = float(pd_cap_slack) + if not np.isfinite(cap_slack) or cap_slack < 0.0: + raise ValueError(f"pd_cap_slack must be finite and nonnegative, got {cap_slack}") + + funded = weights_arr > float(funded_tolerance) + miscoverage = outcomes_arr > high_arr + weighted_outcome = float(weights_arr @ outcomes_arr) + weighted_miscoverage = float(weights_arr @ miscoverage.astype(float)) + weighted_point = float(weights_arr @ point_arr) + weighted_effective = float(weights_arr @ effective_arr) + endpoint_budget = float(weights_arr @ high_arr) + gamma_cp = float(weights_arr @ np.clip(high_arr - point_arr, 0.0, 1.0)) + gamma_internalized = float(weights_arr @ np.clip(effective_arr - point_arr, 0.0, 1.0)) + gamma_residual = float(weights_arr @ np.clip(high_arr - effective_arr, 0.0, 1.0)) + if not np.isclose( + gamma_cp, + gamma_internalized + gamma_residual, + rtol=0.0, + atol=1e-9, + ): + raise ValueError("Conformal-premium decomposition is not internally consistent.") + + effective_cap = tolerance_value + cap_slack + effective_constraint_slack = max(0.0, effective_cap - weighted_effective) + effective_constraint_excess = max(0.0, weighted_effective - effective_cap) + endpoint_budget_upper = effective_cap + gamma_residual + sqrt_alpha = float(np.sqrt(alpha_value)) + empirical_coverage = float(1.0 - miscoverage[funded].mean()) if funded.any() else float("nan") + + return FundedCertificateMetrics( + alpha=alpha_value, + risk_tolerance=tolerance_value, + n_funded=int(funded.sum()), + weighted_outcome=weighted_outcome, + weighted_miscoverage=weighted_miscoverage, + weighted_coverage=1.0 - weighted_miscoverage, + empirical_coverage_funded=empirical_coverage, + weighted_pd_point=weighted_point, + weighted_pd_effective=weighted_effective, + endpoint_budget=endpoint_budget, + gamma_cp=gamma_cp, + gamma_internalized=gamma_internalized, + gamma_residual=gamma_residual, + effective_constraint_slack=effective_constraint_slack, + effective_constraint_excess=effective_constraint_excess, + realized_risk_tolerance_excess=max(0.0, weighted_outcome - tolerance_value), + sqrt_alpha=sqrt_alpha, + endpoint_budget_upper=endpoint_budget_upper, + markov_loss_threshold=endpoint_budget + sqrt_alpha, + markov_loss_cap=endpoint_budget_upper + sqrt_alpha, + ) + + +def add_policy_aware_bound_columns(frame: pd.DataFrame) -> pd.DataFrame: + """Add exact and policy-aware bound columns to alpha-grid evaluation rows. + + Historical pool93 evaluations already store the sufficient statistics for + this decomposition. Rehydrating the bounds from those columns corrects + capped/tail policy semantics without re-solving any portfolio. + """ + required = { + "alpha", + "weighted_pd_high", + "weighted_pd_constraint_used", + } + missing = sorted(required.difference(frame.columns)) + if missing: + raise ValueError(f"Bound-evaluation frame is missing required columns: {missing}") + tolerance_column = "risk_tolerance" if "risk_tolerance" in frame.columns else "tau" + if tolerance_column not in frame.columns: + raise ValueError("Bound-evaluation frame requires risk_tolerance or tau.") + + work = frame.copy() + if "realized_risk_tolerance_excess" not in work.columns and "violation" in work.columns: + work["realized_risk_tolerance_excess"] = pd.to_numeric( + work["violation"], errors="raise" + ).astype(float) + if ( + "empirical_risk_excess_leq_alpha" not in work.columns + and "bound_a_expected_violation_leq_alpha" in work.columns + ): + work["empirical_risk_excess_leq_alpha"] = work[ + "bound_a_expected_violation_leq_alpha" + ].astype(bool) + alpha = pd.to_numeric(work["alpha"], errors="raise").astype(float) + endpoint = pd.to_numeric(work["weighted_pd_high"], errors="raise").astype(float) + effective = pd.to_numeric(work["weighted_pd_constraint_used"], errors="raise").astype(float) + tolerance = pd.to_numeric(work[tolerance_column], errors="raise").astype(float) + slack = ( + pd.to_numeric(work["pd_cap_slack"], errors="raise").astype(float) + if "pd_cap_slack" in work.columns + else pd.Series(0.0, index=work.index, dtype=float) + ) + numeric = pd.concat( + { + "alpha": alpha, + "endpoint": endpoint, + "effective": effective, + "tolerance": tolerance, + "slack": slack, + }, + axis=1, + ) + if not np.isfinite(numeric.to_numpy(dtype=float)).all(): + raise ValueError("Bound-evaluation frame contains non-finite certificate inputs.") + if (alpha <= 0.0).any() or (alpha >= 1.0).any(): + raise ValueError("Bound-evaluation alpha values must lie in (0, 1).") + if (slack < 0.0).any(): + raise ValueError("Bound-evaluation pd_cap_slack must be nonnegative.") + + residual = endpoint - effective + if (residual < -1e-9).any(): + raise ValueError("weighted_pd_high must be at least weighted_pd_constraint_used.") + residual = residual.clip(lower=0.0) + effective_cap = tolerance + slack + sqrt_alpha = np.sqrt(alpha) + work["gamma_residual"] = residual + if "weighted_pd_point" in work.columns: + point = pd.to_numeric(work["weighted_pd_point"], errors="raise").astype(float) + work["gamma_internalized"] = (effective - point).clip(lower=0.0) + elif "gamma_cp" in work.columns: + gamma_cp = pd.to_numeric(work["gamma_cp"], errors="raise").astype(float) + work["gamma_internalized"] = (gamma_cp - residual).clip(lower=0.0) + work["effective_constraint_slack"] = (effective_cap - effective).clip(lower=0.0) + work["effective_constraint_excess"] = (effective - effective_cap).clip(lower=0.0) + work["endpoint_budget"] = endpoint + work["endpoint_budget_upper"] = effective_cap + residual + work["markov_loss_threshold"] = endpoint + sqrt_alpha + work["markov_loss_cap"] = work["endpoint_budget_upper"] + sqrt_alpha + return work diff --git a/src/optimization/cuopt_adapter.py b/src/optimization/cuopt_adapter.py index dbca887..f5e9e57 100644 --- a/src/optimization/cuopt_adapter.py +++ b/src/optimization/cuopt_adapter.py @@ -17,17 +17,17 @@ import numpy as np import pandas as pd from loguru import logger -from scipy.sparse import csr_matrix + +from src.optimization.portfolio_model import _portfolio_lp_components, _PortfolioLpComponents def _require_cuopt() -> Any: try: - from cuopt import linear_programming as lp_api # type: ignore[import-not-found] + return importlib.import_module("cuopt").linear_programming except Exception as exc: # pragma: no cover - exercised in RAPIDS env only raise RuntimeError( "solver_backend='cuopt' requested but native cuOpt Python API is not available." ) from exc - return lp_api def _extract_primal_solution(solution: Any, n_vars: int) -> np.ndarray: @@ -109,112 +109,33 @@ def _unique_cuopt_log_file(log_dir: str | Path, *, random_seed: int | None) -> s return str(target / f"cuopt_seed-{seed_token}_pid-{os.getpid()}_{time.time_ns()}.log") -def solve_portfolio_cuopt_native( - *, - loans: pd.DataFrame, - pd_point: np.ndarray, - pd_high: np.ndarray, - lgd: np.ndarray, - int_rates: np.ndarray, - total_budget: float = 1_000_000, - max_concentration: float = 0.25, - max_portfolio_pd: float = 0.10, - robust: bool = True, - uncertainty_aversion: float = 0.0, - min_budget_utilization: float = 0.0, - pd_cap_slack_penalty: float = 0.0, - pd_constraint_override: np.ndarray | None = None, - time_limit: int = 300, - random_seed: int | None = None, - presolve: int | None = 1, - cuopt_parameters: dict[str, Any] | None = None, -) -> dict[str, Any]: - """Solve the portfolio LP natively with cuOpt.""" - lp_api = _require_cuopt() - - n = len(loans) - if n == 0: - raise ValueError("Cannot solve empty portfolio.") - - loan_amounts = ( - loans["loan_amnt"].to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.ones(n, dtype=float) * 10_000.0 - ) - point = np.asarray(pd_point, dtype=float) - high = np.asarray(pd_high, dtype=float) - lgd_arr = np.asarray(lgd, dtype=float) - rates = np.asarray(int_rates, dtype=float) - pd_constraint = ( - np.asarray(pd_constraint_override, dtype=float) - if pd_constraint_override is not None - else (high if robust else point) - ) - pd_uncertainty = np.clip(high - point, 0.0, 1.0) - - use_pd_slack = float(pd_cap_slack_penalty) > 0 - obj = loan_amounts * (rates - point * lgd_arr - uncertainty_aversion * pd_uncertainty * lgd_arr) - - rows: list[np.ndarray] = [] - rhs: list[float] = [] - row_types: list[str] = [] - - # Budget cap - rows.append(loan_amounts.astype(float)) - rhs.append(float(total_budget)) - row_types.append("L") - - # Optional minimum budget utilization - min_budget_utilization = float(np.clip(min_budget_utilization, 0.0, 1.0)) - if min_budget_utilization > 0: - rows.append((-loan_amounts).astype(float)) - rhs.append(float(-min_budget_utilization * total_budget)) - row_types.append("L") - - # Portfolio PD cap: sum(x_i * loan_i * (pd_i - max_pd)) - slack <= 0 - pd_row = loan_amounts * (pd_constraint - float(max_portfolio_pd)) - rows.append(pd_row.astype(float)) - rhs.append(0.0) - row_types.append("L") - - if "purpose" in loans.columns: - purposes = loans["purpose"].fillna("unknown").astype(str) - top_purposes = purposes.unique() - for purpose in top_purposes: - mask = (purposes == purpose).to_numpy(dtype=float) - row = loan_amounts * (mask - float(max_concentration)) - rows.append(row.astype(float)) - rhs.append(0.0) - row_types.append("L") - - A = np.vstack(rows).astype(np.float64) - - var_lb = np.zeros(n + int(use_pd_slack), dtype=np.float64) - var_ub = np.ones(n + int(use_pd_slack), dtype=np.float64) - if use_pd_slack: - slack_col = np.zeros((A.shape[0], 1), dtype=np.float64) - pd_cap_row_idx = 2 if min_budget_utilization > 0 else 1 - slack_col[pd_cap_row_idx, 0] = -1.0 - A = np.hstack([A, slack_col]) - obj = np.concatenate([obj.astype(np.float64), np.array([-float(pd_cap_slack_penalty)])]) - var_ub[-1] = float(total_budget) - else: - obj = obj.astype(np.float64) - - A_csr = csr_matrix(A) +def _cuopt_data_model(lp_api: Any, components: _PortfolioLpComponents) -> Any: + matrix = components.a_ub.tocsr() dm = lp_api.DataModel() dm.set_csr_constraint_matrix( - A_csr.data.astype(np.float64), - A_csr.indices.astype(np.int32), - A_csr.indptr.astype(np.int32), + matrix.data.astype(np.float64), + matrix.indices.astype(np.int32), + matrix.indptr.astype(np.int32), ) - dm.set_constraint_bounds(np.asarray(rhs, dtype=np.float64)) - dm.set_row_types(np.asarray(row_types)) - dm.set_objective_coefficients(obj) + dm.set_constraint_bounds(components.rhs.astype(np.float64)) + dm.set_row_types(np.asarray(["L"] * len(components.rhs))) + dm.set_objective_coefficients(components.objective_coefficients.astype(np.float64)) dm.set_maximize(True) - dm.set_variable_lower_bounds(var_lb) - dm.set_variable_upper_bounds(var_ub) + bounds = np.asarray(components.bounds, dtype=np.float64) + dm.set_variable_lower_bounds(bounds[:, 0]) + dm.set_variable_upper_bounds(bounds[:, 1]) + return dm + + +def _cuopt_solver_settings( + lp_api: Any, + *, + time_limit: int, + random_seed: int | None, + presolve: int | None, + cuopt_parameters: dict[str, Any] | None, +) -> tuple[Any, dict[str, Any], dict[str, str]]: settings = lp_api.SolverSettings() requested_parameters = { _normalize_parameter_name(k): _coerce_setting_value(v) @@ -237,6 +158,7 @@ def solve_portfolio_cuopt_native( for name, value in requested_parameters.items(): if value is not None: applied_parameters[name] = value + rejected_parameters: dict[str, str] = {} critical_parameters = { "time_limit", @@ -253,19 +175,23 @@ def solve_portfolio_cuopt_native( rejected_parameters[name] = str(exc) if name in critical_parameters: raise + return settings, applied_parameters, rejected_parameters - solution = lp_api.Solve(dm, settings) - termination_reason = str(solution.get_termination_reason()) - if "Optimal" not in termination_reason and "Feasible" not in termination_reason: - raise RuntimeError( - f"cuOpt solve did not produce an acceptable solution: {termination_reason}" - ) - primal = _extract_primal_solution(solution, n + int(use_pd_slack)) - x = primal[:n] - pd_cap_slack = float(primal[-1]) if use_pd_slack else 0.0 - allocation = {i: float(x[i]) for i in range(n)} - total_allocated = float(np.sum(x * loan_amounts)) +def _cuopt_result_payload( + *, + solution: Any, + primal: np.ndarray, + components: _PortfolioLpComponents, + termination_reason: str, + applied_parameters: dict[str, Any], + rejected_parameters: dict[str, str], +) -> dict[str, Any]: + x = np.clip(primal[: components.n], 0.0, 1.0) + pd_cap_slack = float(primal[-1]) if components.use_pd_slack else 0.0 + allocation = {i: float(x[i]) for i in range(components.n)} + allocation_vector = x.astype(float) + total_allocated = float(np.sum(x * components.loan_amounts)) n_funded = int(np.sum(x > 0.01)) obj_value = float(solution.get_primal_objective()) @@ -273,13 +199,14 @@ def solve_portfolio_cuopt_native( "Portfolio solved (cuopt_native): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, pd_cap_slack={:.4f}", obj_value, n_funded, - n, + components.n, total_allocated, pd_cap_slack, ) return { "allocation": allocation, + "allocation_vector": allocation_vector, "objective_value": obj_value, "n_funded": n_funded, "total_allocated": total_allocated, @@ -292,3 +219,67 @@ def solve_portfolio_cuopt_native( "cuopt_log_file": applied_parameters.get("log_file", ""), "cuopt_rejected_parameters": rejected_parameters, } + + +def solve_portfolio_cuopt_native( + *, + loans: pd.DataFrame, + pd_point: np.ndarray, + pd_high: np.ndarray, + lgd: np.ndarray, + int_rates: np.ndarray, + total_budget: float = 1_000_000, + max_concentration: float = 0.25, + max_portfolio_pd: float = 0.10, + robust: bool = True, + uncertainty_aversion: float = 0.0, + min_budget_utilization: float = 0.0, + pd_cap_slack_penalty: float = 0.0, + pd_constraint_override: np.ndarray | None = None, + time_limit: int = 300, + random_seed: int | None = None, + presolve: int | None = 1, + cuopt_parameters: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Solve the portfolio LP natively with cuOpt.""" + lp_api = _require_cuopt() + components = _portfolio_lp_components( + loans=loans, + pd_point=pd_point, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=max_portfolio_pd, + robust=robust, + uncertainty_aversion=uncertainty_aversion, + min_budget_utilization=min_budget_utilization, + pd_cap_slack_penalty=pd_cap_slack_penalty, + pd_constraint_override=pd_constraint_override, + ) + dm = _cuopt_data_model(lp_api, components) + settings, applied_parameters, rejected_parameters = _cuopt_solver_settings( + lp_api, + time_limit=time_limit, + random_seed=random_seed, + presolve=presolve, + cuopt_parameters=cuopt_parameters, + ) + + solution = lp_api.Solve(dm, settings) + termination_reason = str(solution.get_termination_reason()) + if "Optimal" not in termination_reason and "Feasible" not in termination_reason: + raise RuntimeError( + f"cuOpt solve did not produce an acceptable solution: {termination_reason}" + ) + + primal = _extract_primal_solution(solution, components.n + int(components.use_pd_slack)) + return _cuopt_result_payload( + solution=solution, + primal=primal, + components=components, + termination_reason=termination_reason, + applied_parameters=applied_parameters, + rejected_parameters=rejected_parameters, + ) diff --git a/src/optimization/input_alignment.py b/src/optimization/input_alignment.py new file mode 100644 index 0000000..2091af6 --- /dev/null +++ b/src/optimization/input_alignment.py @@ -0,0 +1,195 @@ +"""Deterministic alignment of candidate loans and conformal interval rows.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +import numpy as np +import pandas as pd + +AlignmentMode = Literal["id", "row_number", "position"] + + +@dataclass(frozen=True) +class AlignedCandidateIntervals: + """Candidate and interval frames that share the same sampled row order.""" + + candidates: pd.DataFrame + intervals: pd.DataFrame + mode: AlignmentMode + available_rows: int + + @property + def selected_rows(self) -> int: + return len(self.candidates) + + +def _normalized_limit(max_candidates: int | None) -> int | None: + if max_candidates is None or int(max_candidates) <= 0: + return None + return int(max_candidates) + + +def _sample_positions( + row_count: int, + *, + max_candidates: int | None, + random_state: int, +) -> np.ndarray: + if row_count <= 0: + raise ValueError("Candidate/interval alignment produced zero rows.") + limit = _normalized_limit(max_candidates) + sample_size = row_count if limit is None else min(row_count, limit) + if sample_size == row_count: + return np.arange(row_count, dtype=int) + positions = np.random.default_rng(random_state).choice( + row_count, + size=sample_size, + replace=False, + ) + return np.sort(positions.astype(int, copy=False)) + + +def _require_unique_columns(frame: pd.DataFrame, *, source: str) -> None: + duplicated = frame.columns[frame.columns.duplicated()].tolist() + if duplicated: + raise ValueError(f"{source} contains duplicate column names: {duplicated}") + + +def _validated_string_key(series: pd.Series, *, source: str) -> pd.Series: + if series.isna().any(): + raise ValueError(f"{source} alignment key contains missing values.") + keys = series.astype(str) + if keys.str.strip().eq("").any(): + raise ValueError(f"{source} alignment key contains blank values.") + duplicated = keys[keys.duplicated(keep=False)] + if not duplicated.empty: + examples = duplicated.drop_duplicates().head(3).tolist() + raise ValueError(f"{source} alignment key is not unique; examples={examples}") + return keys + + +def _validated_row_number_key(series: pd.Series) -> pd.Series: + if series.isna().any(): + raise ValueError("interval _row_number contains missing values.") + numeric = pd.to_numeric(series, errors="raise") + values = numeric.to_numpy(dtype=float) + if not np.isfinite(values).all() or not np.equal(values, np.floor(values)).all(): + raise ValueError("interval _row_number must contain finite integers.") + keys = pd.Series(values.astype(np.int64), index=series.index) + duplicated = keys[keys.duplicated(keep=False)] + if not duplicated.empty: + examples = duplicated.drop_duplicates().head(3).tolist() + raise ValueError(f"interval _row_number is not unique; examples={examples}") + return keys + + +def _unused_column_name(base: str, used: set[str]) -> str: + name = base + suffix = 1 + while name in used: + name = f"{base}_{suffix}" + suffix += 1 + used.add(name) + return name + + +def _align_by_key( + candidates: pd.DataFrame, + intervals: pd.DataFrame, + *, + candidate_keys: pd.Series, + interval_keys: pd.Series, + mode: AlignmentMode, + max_candidates: int | None, + random_state: int, +) -> AlignedCandidateIntervals: + used_columns = set(candidates.columns) | set(intervals.columns) + join_column = _unused_column_name("__crpto_alignment_key", used_columns) + interval_aliases = { + column: _unused_column_name(f"__crpto_interval_{index}", used_columns) + for index, column in enumerate(intervals.columns) + } + + candidate_work = candidates.copy() + candidate_work[join_column] = candidate_keys.to_numpy() + interval_work = intervals.rename(columns=interval_aliases).copy() + interval_work[join_column] = interval_keys.to_numpy() + merged = candidate_work.merge( + interval_work, + on=join_column, + how="inner", + sort=False, + validate="one_to_one", + ) + positions = _sample_positions( + len(merged), + max_candidates=max_candidates, + random_state=random_state, + ) + sampled = merged.iloc[positions].reset_index(drop=True) + aligned_candidates = sampled.loc[:, list(candidates.columns)].copy() + aligned_intervals = sampled.loc[:, list(interval_aliases.values())].rename( + columns={alias: source for source, alias in interval_aliases.items()} + ) + return AlignedCandidateIntervals( + candidates=aligned_candidates, + intervals=aligned_intervals, + mode=mode, + available_rows=len(merged), + ) + + +def align_candidate_intervals( + candidates: pd.DataFrame, + intervals: pd.DataFrame, + *, + max_candidates: int | None, + random_state: int, + allow_row_number: bool = True, +) -> AlignedCandidateIntervals: + """Align and sample candidate/interval rows under a strict one-to-one contract. + + Stable IDs take precedence. Legacy interval artifacts may instead provide a + zero-based ``_row_number``. A positional fallback is retained for older + artifacts, but samples from the full alignable universe rather than taking + a deterministic prefix. + """ + _require_unique_columns(candidates, source="candidates") + _require_unique_columns(intervals, source="intervals") + + if "id" in candidates.columns and "id" in intervals.columns: + return _align_by_key( + candidates, + intervals, + candidate_keys=_validated_string_key(candidates["id"], source="candidate id"), + interval_keys=_validated_string_key(intervals["id"], source="interval id"), + mode="id", + max_candidates=max_candidates, + random_state=random_state, + ) + + if allow_row_number and "_row_number" in intervals.columns: + return _align_by_key( + candidates, + intervals, + candidate_keys=pd.Series(np.arange(len(candidates), dtype=np.int64)), + interval_keys=_validated_row_number_key(intervals["_row_number"]), + mode="row_number", + max_candidates=max_candidates, + random_state=random_state, + ) + + available_rows = min(len(candidates), len(intervals)) + positions = _sample_positions( + available_rows, + max_candidates=max_candidates, + random_state=random_state, + ) + return AlignedCandidateIntervals( + candidates=candidates.iloc[positions].reset_index(drop=True).copy(), + intervals=intervals.iloc[positions].reset_index(drop=True).copy(), + mode="position", + available_rows=available_rows, + ) diff --git a/src/optimization/policy.py b/src/optimization/policy.py index 89303ea..408f5c6 100644 --- a/src/optimization/policy.py +++ b/src/optimization/policy.py @@ -10,6 +10,10 @@ from __future__ import annotations from enum import StrEnum +from typing import cast + +import numpy as np +import pandas as pd class PolicyMode(StrEnum): @@ -79,3 +83,44 @@ def resolve_policy_mode(value: str | PolicyMode | None) -> PolicyMode: def all_policy_modes() -> tuple[PolicyMode, ...]: """Tuple of all canonical policy modes — useful for parametrised tests and sweeps.""" return tuple(PolicyMode) + + +_SEGMENT_POLICY_MODES = frozenset( + { + PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY, + PolicyMode.SEGMENT_RELATIVE_TAIL_BLENDED_UNCERTAINTY, + } +) + + +def policy_uses_segment_labels(value: str | PolicyMode | None) -> bool: + """Return whether a policy requires contextual segment labels.""" + return resolve_policy_mode(value) in _SEGMENT_POLICY_MODES + + +def policy_segment_labels( + loans: pd.DataFrame, + policy_mode: str | PolicyMode | None, + *, + grade_column: str = "grade", +) -> np.ndarray | None: + """Build the canonical grade/term/verification labels for segment policies. + + Non-segment policies return ``None``. Missing context columns are represented + by ``"unknown"`` so all optimization entrypoints use the same fallback. + """ + if not policy_uses_segment_labels(policy_mode): + return None + + def _labels(column: str) -> pd.Series: + if column not in loans.columns: + return pd.Series("unknown", index=loans.index, dtype="string") + return loans[column].fillna("unknown").astype(str) + + grade = _labels(grade_column) + term = _labels("term") + verification = _labels("verification_status") + return cast( + np.ndarray, + (grade + "|" + term + "|" + verification).to_numpy(dtype=object), + ) diff --git a/src/optimization/policy_evaluation.py b/src/optimization/policy_evaluation.py new file mode 100644 index 0000000..43e69e2 --- /dev/null +++ b/src/optimization/policy_evaluation.py @@ -0,0 +1,109 @@ +"""Shared execution contract for portfolio uncertainty policies.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + +from src.optimization.policy import PolicyMode, policy_segment_labels, resolve_policy_mode +from src.optimization.portfolio_model import ( + compute_effective_pd, + optimize_portfolio_allocation, + solution_allocation_vector, +) + + +@dataclass(frozen=True) +class PolicyAllocationResult: + """Solver payload, dense allocation, and effective PD for one policy.""" + + solution: dict[str, Any] + allocation: np.ndarray + effective_pd: np.ndarray + policy_mode: PolicyMode + gamma: float + delta_cap_quantile: float + tail_focus_quantile: float + objective_risk_mode: str + + +def solve_policy_allocation( + *, + loans: pd.DataFrame, + pd_point: np.ndarray, + pd_low: np.ndarray, + pd_high: np.ndarray, + lgd: np.ndarray, + int_rates: np.ndarray, + total_budget: float = 1_000_000.0, + max_concentration: float = 0.25, + risk_tolerance: float = 0.10, + robust: bool = True, + uncertainty_aversion: float = 0.0, + min_budget_utilization: float = 0.0, + pd_cap_slack_penalty: float = 0.0, + policy_mode: str | PolicyMode = PolicyMode.HARD_WORST_CASE, + gamma: float = 1.0, + delta_cap_quantile: float = 1.0, + tail_focus_quantile: float = 1.0, + time_limit: int = 300, + threads: int = 4, + solver_backend: str = "highs", + random_seed: int | None = None, + cuopt_presolve: int | None = 1, + cuopt_parameters: dict[str, Any] | None = None, +) -> PolicyAllocationResult: + """Resolve policy semantics once and solve the corresponding portfolio. + + ``robust=False`` always means a point-PD constraint, regardless of any + uncertainty-policy arguments supplied by a legacy caller. + """ + effective_mode = resolve_policy_mode(policy_mode) if robust else PolicyMode.POINT_ESTIMATE + effective_gamma = float(gamma) if robust else 0.0 + effective_delta_cap = float(delta_cap_quantile) if robust else 1.0 + effective_tail_focus = float(tail_focus_quantile) if robust else 1.0 + effective_aversion = float(uncertainty_aversion) if robust else 0.0 + effective_pd = compute_effective_pd( + pd_point=pd_point, + pd_high=pd_high, + policy_mode=effective_mode, + gamma=effective_gamma, + delta_cap_quantile=effective_delta_cap, + tail_focus_quantile=effective_tail_focus, + segment_labels=policy_segment_labels(loans, effective_mode), + ) + solution = optimize_portfolio_allocation( + loans=loans, + pd_point=pd_point, + pd_low=pd_low, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=risk_tolerance, + robust=robust, + uncertainty_aversion=effective_aversion, + min_budget_utilization=min_budget_utilization, + pd_cap_slack_penalty=pd_cap_slack_penalty, + pd_constraint_override=effective_pd, + time_limit=time_limit, + threads=threads, + solver_backend=solver_backend, + random_seed=random_seed, + cuopt_presolve=cuopt_presolve, + cuopt_parameters=cuopt_parameters, + ) + return PolicyAllocationResult( + solution=solution, + allocation=solution_allocation_vector(solution, len(loans)), + effective_pd=effective_pd, + policy_mode=effective_mode, + gamma=effective_gamma, + delta_cap_quantile=effective_delta_cap, + tail_focus_quantile=effective_tail_focus, + objective_risk_mode="point_pd_plus_aversion", + ) diff --git a/src/optimization/policy_selection.py b/src/optimization/policy_selection.py new file mode 100644 index 0000000..a8b3d7f --- /dev/null +++ b/src/optimization/policy_selection.py @@ -0,0 +1,228 @@ +"""Simple, outcome-free selection primitives for the active IJDS policy.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from itertools import product + +import numpy as np +import pandas as pd + +from src.optimization.policy import PolicyMode + +FORBIDDEN_POLICY_SELECTION_COLUMNS = frozenset( + { + "default_flag", + "y_true", + "outcome", + "weighted_outcome", + "weighted_miscoverage", + "realized_return", + "realized_risk_tolerance_excess", + } +) +REQUIRED_POLICY_SELECTION_COLUMNS = frozenset( + { + "candidate_id", + "solver_status", + "expected_objective", + "endpoint_budget", + "weighted_pd_effective", + "risk_tolerance", + "total_allocated", + } +) + + +@dataclass(frozen=True) +class LinearPolicyCandidate: + """One linear conformal-guardrail policy.""" + + candidate_id: str + risk_tolerance: float + gamma: float + uncertainty_aversion: float + policy_mode: str = PolicyMode.BLENDED_UNCERTAINTY.value + delta_cap_quantile: float = 1.0 + tail_focus_quantile: float = 1.0 + min_budget_utilization: float = 0.0 + pd_cap_slack_penalty: float = 0.0 + + def to_record(self) -> dict[str, float | str]: + """Return a JSON/table-friendly record.""" + return asdict(self) + + +def build_linear_policy_grid( + *, + risk_tolerances: list[float], + gammas: list[float], + uncertainty_aversions: list[float], +) -> list[LinearPolicyCandidate]: + """Build a deterministic Cartesian grid of linear policies.""" + return [ + LinearPolicyCandidate( + candidate_id=f"linear-{index:03d}", + risk_tolerance=float(tau), + gamma=float(gamma), + uncertainty_aversion=float(aversion), + ) + for index, (tau, gamma, aversion) in enumerate( + product(risk_tolerances, gammas, uncertainty_aversions), + start=1, + ) + ] + + +def temporal_period_labels( + issue_dates: pd.Series, + *, + combine_years_from: int | None = 2020, +) -> pd.Series: + """Map issue dates to half-year periods, optionally pooling late years.""" + dates = pd.to_datetime(issue_dates, errors="coerce") + if dates.isna().any(): + raise ValueError(f"issue_dates contains {int(dates.isna().sum())} invalid values.") + years = dates.dt.year.astype(int) + halves = np.where(dates.dt.month.to_numpy() <= 6, "H1", "H2") + labels = pd.Series(years.astype(str) + halves, index=issue_dates.index, dtype="string") + if combine_years_from is not None: + labels.loc[years >= int(combine_years_from)] = f"{int(combine_years_from)}+" + return labels + + +def policy_eligibility_mask( + results: pd.DataFrame, + *, + endpoint_budget_cap: float, + budget: float, + min_budget_utilization: float = 0.999, +) -> pd.Series: + """Return the deterministic ex-ante feasibility screen for policy rows.""" + missing = sorted(REQUIRED_POLICY_SELECTION_COLUMNS.difference(results.columns)) + if missing: + raise ValueError(f"Policy results are missing required columns: {missing}") + if float(budget) <= 0.0: + raise ValueError("budget must be positive.") + if not 0.0 <= float(min_budget_utilization) <= 1.0: + raise ValueError("min_budget_utilization must lie in [0, 1].") + + solver_ok = results["solver_status"].astype(str).str.strip().str.casefold().eq("optimal") + budget_ok = pd.to_numeric(results["total_allocated"], errors="raise") >= ( + float(budget) * float(min_budget_utilization) + ) + endpoint_ok = pd.to_numeric(results["endpoint_budget"], errors="raise") <= ( + float(endpoint_budget_cap) + 1e-12 + ) + cap_ok = pd.to_numeric(results["weighted_pd_effective"], errors="raise") <= ( + pd.to_numeric(results["risk_tolerance"], errors="raise") + 1e-12 + ) + objective_ok = np.isfinite( + pd.to_numeric(results["expected_objective"], errors="raise").to_numpy(dtype=float) + ) + return solver_ok & budget_ok & endpoint_ok & cap_ok & objective_ok + + +def endpoint_cap_stability( + results: pd.DataFrame, + *, + selected_candidate_id: str, + endpoint_budget_cap: float, + budget: float, + min_budget_utilization: float = 0.999, +) -> dict[str, float | str | None]: + """Return the cap interval over which the selected candidate stays optimal. + + Solver, budget, effective-PD, and finite-objective screens are held fixed. + Eligibility is monotone in the endpoint cap, so a higher-ranked candidate + can displace the selected policy only when its endpoint first becomes + feasible. + """ + base_feasible = results.loc[ + policy_eligibility_mask( + results, + endpoint_budget_cap=float("inf"), + budget=budget, + min_budget_utilization=min_budget_utilization, + ) + ].copy() + selected_rows = base_feasible.loc[ + base_feasible["candidate_id"].astype(str).eq(str(selected_candidate_id)) + ] + if len(selected_rows) != 1: + raise ValueError( + "selected_candidate_id must identify exactly one base-feasible policy row." + ) + ranked = base_feasible.sort_values( + ["expected_objective", "endpoint_budget", "candidate_id"], + ascending=[False, True, True], + kind="mergesort", + ).reset_index(drop=True) + selected_position = int( + ranked.index[ranked["candidate_id"].astype(str).eq(str(selected_candidate_id))][0] + ) + selected_endpoint = float(selected_rows.iloc[0]["endpoint_budget"]) + superior = ranked.iloc[:selected_position] + upper_exclusive = ( + float(pd.to_numeric(superior["endpoint_budget"], errors="raise").min()) + if not superior.empty + else None + ) + cap = float(endpoint_budget_cap) + if cap + 1e-12 < selected_endpoint or ( + upper_exclusive is not None and cap + 1e-12 >= upper_exclusive + ): + raise ValueError("Selected candidate is not optimal at endpoint_budget_cap.") + return { + "selected_candidate_id": str(selected_candidate_id), + "selected_endpoint_budget": selected_endpoint, + "cap_lower_inclusive": selected_endpoint, + "cap_upper_exclusive": upper_exclusive, + "declared_endpoint_budget_cap": cap, + "margin_to_lower_boundary": cap - selected_endpoint, + "margin_to_upper_boundary": (None if upper_exclusive is None else upper_exclusive - cap), + } + + +def select_policy_result_ex_ante( + results: pd.DataFrame, + *, + endpoint_budget_cap: float, + budget: float, + min_budget_utilization: float = 0.999, +) -> tuple[pd.Series, dict[str, int | float | str]]: + """Maximize expected objective under outcome-free feasibility screens.""" + forbidden = sorted(FORBIDDEN_POLICY_SELECTION_COLUMNS.intersection(results.columns)) + if forbidden: + raise ValueError(f"Ex-ante selector received outcome-derived columns: {forbidden}") + if results.empty: + raise ValueError("Ex-ante selection results are empty.") + if results["candidate_id"].astype(str).duplicated().any(): + raise ValueError("Ex-ante selection results contain duplicate candidate_id values.") + eligible_mask = policy_eligibility_mask( + results, + endpoint_budget_cap=endpoint_budget_cap, + budget=budget, + min_budget_utilization=min_budget_utilization, + ) + eligible = results.loc[eligible_mask].copy() + if eligible.empty: + raise RuntimeError( + "No policy satisfies the deterministic endpoint, effective-PD, and budget screens." + ) + selected = eligible.sort_values( + ["expected_objective", "endpoint_budget", "candidate_id"], + ascending=[False, True, True], + kind="mergesort", + ).iloc[0] + audit: dict[str, int | float | str] = { + "selection_rule": "max_expected_objective_under_deterministic_endpoint_screen", + "n_total": int(len(results)), + "n_eligible": int(len(eligible)), + "selected_candidate_id": str(selected["candidate_id"]), + "endpoint_budget_cap": float(endpoint_budget_cap), + "min_budget_utilization": float(min_budget_utilization), + "outcome_columns_used": 0, + "statistical_assumption_columns_used": 0, + } + return selected, audit diff --git a/src/optimization/portfolio_model.py b/src/optimization/portfolio_model.py index ce180f6..ebc9138 100644 --- a/src/optimization/portfolio_model.py +++ b/src/optimization/portfolio_model.py @@ -10,6 +10,8 @@ from __future__ import annotations import os +from collections.abc import Mapping +from dataclasses import dataclass from typing import Any, cast import numpy as np @@ -22,6 +24,45 @@ from src.optimization.policy import PolicyMode, resolve_policy_mode +@dataclass(frozen=True) +class _PortfolioLpComponents: + n: int + loan_amounts: np.ndarray + objective_coefficients: np.ndarray + a_ub: csr_matrix + rhs: np.ndarray + bounds: list[tuple[float, float]] + use_pd_slack: bool + + +def solution_allocation_vector(solution: Mapping[str, Any], n_items: int) -> np.ndarray: + """Return a validated dense allocation vector from any solver payload.""" + n = int(n_items) + if n < 0: + raise ValueError(f"n_items must be nonnegative, got {n}") + + raw_vector = solution.get("allocation_vector") + if raw_vector is not None: + allocation = np.asarray(raw_vector, dtype=float) + else: + raw_mapping = solution.get("allocation") + if not isinstance(raw_mapping, Mapping): + raise TypeError( + "Solver result must contain allocation_vector or an allocation mapping." + ) + allocation = np.fromiter( + (float(raw_mapping.get(i, 0.0)) for i in range(n)), + dtype=float, + count=n, + ) + + if allocation.shape != (n,): + raise ValueError(f"Solver allocation shape mismatch: {allocation.shape} != {(n,)}") + if not np.all(np.isfinite(allocation)): + raise ValueError("Solver allocation contains non-finite values.") + return allocation + + def compute_effective_pd( pd_point: np.ndarray, pd_high: np.ndarray, @@ -61,71 +102,93 @@ def compute_effective_pd( if mode is PolicyMode.HARD_WORST_CASE: return high if mode is PolicyMode.BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - return cast( - np.ndarray, - np.clip(point + weight * np.clip(high - point, 0.0, 1.0), 0.0, 1.0), - ) + return _blend_pd(point, _pd_delta(point, high), gamma) if mode is PolicyMode.CAPPED_BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - delta = np.clip(high - point, 0.0, 1.0) - q = 1.0 if delta_cap_quantile is None else float(np.clip(delta_cap_quantile, 0.0, 1.0)) - delta_cap = float(np.quantile(delta, q)) if len(delta) else 0.0 - return cast(np.ndarray, np.clip(point + weight * np.minimum(delta, delta_cap), 0.0, 1.0)) + delta = _pd_delta(point, high) + cap = _quantile_or_zero(delta, _clipped_quantile(delta_cap_quantile, default=1.0)) + return _blend_pd(point, np.minimum(delta, cap), gamma) if mode is PolicyMode.TAIL_BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - delta = np.clip(high - point, 0.0, 1.0) - q = 0.9 if tail_focus_quantile is None else float(np.clip(tail_focus_quantile, 0.0, 1.0)) - cutoff = float(np.quantile(delta, q)) if len(delta) else 0.0 - local_delta = np.where(delta >= cutoff, delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) + delta = _pd_delta(point, high) + local_delta = _tail_delta( + delta=delta, + score=delta, + q=_clipped_quantile(tail_focus_quantile, default=0.9), + ) + return _blend_pd(point, local_delta, gamma) if mode is PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - delta = np.clip(high - point, 0.0, 1.0) - q = 0.9 if tail_focus_quantile is None else float(np.clip(tail_focus_quantile, 0.0, 1.0)) - global_cutoff = float(np.quantile(delta, q)) if len(delta) else 0.0 - if segment_labels is None or len(segment_labels) != len(delta): - local_delta = np.where(delta >= global_cutoff, delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) - - labels = pd.Series(np.asarray(segment_labels, dtype=object)).fillna("unknown").astype(str) - local_delta = np.zeros_like(delta) - for label in labels.unique(): - mask = labels == label - mask_arr = mask.to_numpy(dtype=bool) - seg_delta = delta[mask_arr] - if len(seg_delta) < int(max(min_segment_size, 1)): - cutoff = global_cutoff - else: - cutoff = float(np.quantile(seg_delta, q)) - local_delta[mask_arr] = np.where(seg_delta >= cutoff, seg_delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) + delta = _pd_delta(point, high) + q = _clipped_quantile(tail_focus_quantile, default=0.9) + local_delta = _segment_tail_delta( + delta=delta, + score=delta, + q=q, + segment_labels=segment_labels, + min_segment_size=min_segment_size, + ) + return _blend_pd(point, local_delta, gamma) if mode is PolicyMode.SEGMENT_RELATIVE_TAIL_BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - delta = np.clip(high - point, 0.0, 1.0) + delta = _pd_delta(point, high) relative_width = delta / np.maximum(point, 1e-4) - q = 0.9 if tail_focus_quantile is None else float(np.clip(tail_focus_quantile, 0.0, 1.0)) - global_cutoff = float(np.quantile(relative_width, q)) if len(relative_width) else 0.0 - if segment_labels is None or len(segment_labels) != len(delta): - local_delta = np.where(relative_width >= global_cutoff, delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) - - labels = pd.Series(np.asarray(segment_labels, dtype=object)).fillna("unknown").astype(str) - local_delta = np.zeros_like(delta) - for label in labels.unique(): - mask = labels == label - mask_arr = mask.to_numpy(dtype=bool) - seg_delta = delta[mask_arr] - seg_relative = relative_width[mask_arr] - if len(seg_relative) < int(max(min_segment_size, 1)): - cutoff = global_cutoff - else: - cutoff = float(np.quantile(seg_relative, q)) - local_delta[mask_arr] = np.where(seg_relative >= cutoff, seg_delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) + q = _clipped_quantile(tail_focus_quantile, default=0.9) + local_delta = _segment_tail_delta( + delta=delta, + score=relative_width, + q=q, + segment_labels=segment_labels, + min_segment_size=min_segment_size, + ) + return _blend_pd(point, local_delta, gamma) raise ValueError(f"Unhandled policy mode: {mode!r}") # safety net, unreachable +def _pd_delta(point: np.ndarray, high: np.ndarray) -> np.ndarray: + return cast(np.ndarray, np.clip(high - point, 0.0, 1.0)) + + +def _clipped_quantile(value: float | None, *, default: float) -> float: + raw = default if value is None else float(value) + return float(np.clip(raw, 0.0, 1.0)) + + +def _quantile_or_zero(values: np.ndarray, q: float) -> float: + return float(np.quantile(values, q)) if len(values) else 0.0 + + +def _blend_pd(point: np.ndarray, delta: np.ndarray, gamma: float) -> np.ndarray: + weight = float(np.clip(gamma, 0.0, 1.0)) + return np.clip(point + weight * delta, 0.0, 1.0) + + +def _tail_delta(*, delta: np.ndarray, score: np.ndarray, q: float) -> np.ndarray: + cutoff = _quantile_or_zero(score, q) + return np.where(score >= cutoff, delta, 0.0) + + +def _segment_tail_delta( + *, + delta: np.ndarray, + score: np.ndarray, + q: float, + segment_labels: np.ndarray | None, + min_segment_size: int, +) -> np.ndarray: + if segment_labels is None or len(segment_labels) != len(delta): + return _tail_delta(delta=delta, score=score, q=q) + + labels = pd.Series(np.asarray(segment_labels, dtype=object)).fillna("unknown").astype(str) + local_delta = np.zeros_like(delta) + global_cutoff = _quantile_or_zero(score, q) + min_size = int(max(min_segment_size, 1)) + for label in labels.unique(): + mask_arr = (labels == label).to_numpy(dtype=bool) + segment_score = score[mask_arr] + cutoff = ( + global_cutoff if len(segment_score) < min_size else _quantile_or_zero(segment_score, q) + ) + local_delta[mask_arr] = np.where(segment_score >= cutoff, delta[mask_arr], 0.0) + return local_delta + + def build_portfolio_model( loans: pd.DataFrame, pd_point: np.ndarray, @@ -264,49 +327,102 @@ def solve_portfolio( ) -> dict[str, Any]: """Solve portfolio optimization with HiGHS (default) or optional cuOpt.""" backend = solver_backend.strip().lower() + results = _solve_pyomo_backend( + model, + backend=backend, + time_limit=time_limit, + threads=threads, + ) + solution = _pyomo_portfolio_solution(model, backend=backend, results=results) + logger.info( + "Portfolio solved ({}): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, pd_cap_slack={:.4f}", + backend, + solution["objective_value"], + solution["n_funded"], + len(solution["allocation"]), + solution["total_allocated"], + solution["pd_cap_slack"], + ) + return solution + + +def _solve_pyomo_backend( + model: pyo.ConcreteModel, + *, + backend: str, + time_limit: int, + threads: int, +) -> Any: if backend == "highs": - from pyomo.contrib.appsi.solvers import Highs - - solver = Highs() - solver.config.time_limit = time_limit - _ = threads # reserved for future HiGHS appsi configuration - results = solver.solve(model) - elif backend == "cuopt": - solver = pyo.SolverFactory("cuopt") - if solver is None or not solver.available(False): - raise RuntimeError( - "solver_backend='cuopt' requested but Pyomo cuOpt solver is not available " - "in this environment." - ) - _ = (time_limit, threads) # backend-specific options vary by cuOpt deployment - results = solver.solve(model) - else: - raise ValueError(f"Unsupported solver_backend={solver_backend!r}. Use 'highs' or 'cuopt'.") + return _solve_pyomo_highs(model, time_limit=time_limit, threads=threads) + if backend == "cuopt": + return _solve_pyomo_cuopt(model, time_limit=time_limit, threads=threads) + raise ValueError(f"Unsupported solver_backend={backend!r}. Use 'highs' or 'cuopt'.") + + +def _solve_pyomo_highs( + model: pyo.ConcreteModel, + *, + time_limit: int, + threads: int, +) -> Any: + from pyomo.contrib.appsi.solvers import Highs + + solver = Highs() + solver.config.time_limit = time_limit + _ = threads # reserved for future HiGHS appsi configuration + return solver.solve(model) - allocation = {i: pyo.value(model.x[i]) for i in model.I} - obj_value = pyo.value(model.obj) + +def _solve_pyomo_cuopt( + model: pyo.ConcreteModel, + *, + time_limit: int, + threads: int, +) -> Any: + solver = pyo.SolverFactory("cuopt") + if solver is None or not solver.available(False): + raise RuntimeError( + "solver_backend='cuopt' requested but Pyomo cuOpt solver is not available " + "in this environment." + ) + _ = (time_limit, threads) # backend-specific options vary by cuOpt deployment + return solver.solve(model) + + +def _pyomo_portfolio_solution( + model: pyo.ConcreteModel, + *, + backend: str, + results: Any, +) -> dict[str, Any]: + index_set = cast(Any, model.I) + decision_vars = cast(Any, model.x) + loan_amount_param = cast(Any, model.loan_amnt) + allocation = {i: pyo.value(decision_vars[i]) for i in index_set} + obj_value = pyo.value(cast(Any, model.obj)) n_funded = sum(1 for v in allocation.values() if v > 0.01) - total_allocated = sum(allocation[i] * pyo.value(model.loan_amnt[i]) for i in model.I) - pd_cap_slack = float(pyo.value(model.pd_cap_slack)) if hasattr(model, "pd_cap_slack") else 0.0 - termination = getattr(results, "termination_condition", None) - if termination is None and hasattr(results, "solver"): - termination = getattr(results.solver, "termination_condition", None) + total_allocated = sum(allocation[i] * pyo.value(loan_amount_param[i]) for i in index_set) + pd_cap_slack = ( + float(pyo.value(cast(Any, model.pd_cap_slack))) if hasattr(model, "pd_cap_slack") else 0.0 + ) - solution = { + return { "allocation": allocation, "objective_value": float(obj_value), "n_funded": int(n_funded), "total_allocated": float(total_allocated), - "solver_status": str(termination) if termination is not None else "unknown", + "solver_status": _pyomo_termination_status(results), "solver_backend": backend, "pd_cap_slack": pd_cap_slack, } - logger.info( - f"Portfolio solved ({backend}): obj={obj_value:,.2f}, funded={n_funded}/{len(allocation)}, " - f"allocated={total_allocated:,.0f}, pd_cap_slack={pd_cap_slack:.4f}" - ) - return solution + +def _pyomo_termination_status(results: Any) -> str: + termination = getattr(results, "termination_condition", None) + if termination is None and hasattr(results, "solver"): + termination = getattr(results.solver, "termination_condition", None) + return str(termination) if termination is not None else "unknown" def solve_portfolio_highs_sparse( @@ -334,62 +450,21 @@ def solve_portfolio_highs_sparse( model construction. The CRPTO exact rerank solves thousands of very similar large LPs, so avoiding per-check symbolic model creation is material. """ - n = len(loans) - if n == 0: - raise ValueError("Cannot solve empty portfolio.") - - loan_amounts = ( - pd.to_numeric(loans["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.ones(n, dtype=float) - ) - point = np.asarray(pd_point, dtype=float) - high = np.asarray(pd_high, dtype=float) - lgd_arr = np.asarray(lgd, dtype=float) - rates = np.asarray(int_rates, dtype=float) - pd_constraint = ( - np.asarray(pd_constraint_override, dtype=float) - if pd_constraint_override is not None - else (high if robust else point) - ) - pd_uncertainty = np.clip(high - point, 0.0, 1.0) - - objective = loan_amounts * ( - rates - point * lgd_arr - float(uncertainty_aversion) * pd_uncertainty * lgd_arr + components = _portfolio_lp_components( + loans=loans, + pd_point=pd_point, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=max_portfolio_pd, + robust=robust, + uncertainty_aversion=uncertainty_aversion, + min_budget_utilization=min_budget_utilization, + pd_cap_slack_penalty=pd_cap_slack_penalty, + pd_constraint_override=pd_constraint_override, ) - use_pd_slack = float(pd_cap_slack_penalty) > 0.0 - - rows: list[np.ndarray] = [] - rhs: list[float] = [] - - rows.append(loan_amounts.astype(float)) - rhs.append(float(total_budget)) - - min_budget_utilization = float(np.clip(min_budget_utilization, 0.0, 1.0)) - if min_budget_utilization > 0: - rows.append((-loan_amounts).astype(float)) - rhs.append(float(-min_budget_utilization * total_budget)) - - pd_cap_row_idx = len(rows) - rows.append((loan_amounts * (pd_constraint - float(max_portfolio_pd))).astype(float)) - rhs.append(0.0) - - if "purpose" in loans.columns: - purposes = loans["purpose"].fillna("unknown").astype(str) - for purpose in purposes.unique(): - mask = (purposes == purpose).to_numpy(dtype=float) - rows.append((loan_amounts * (mask - float(max_concentration))).astype(float)) - rhs.append(0.0) - - A_ub = csr_matrix(np.vstack(rows).astype(float)) - c = -objective.astype(float) - bounds: list[tuple[float, float | None]] = [(0.0, 1.0)] * n - if use_pd_slack: - slack_col = np.zeros((A_ub.shape[0], 1), dtype=float) - slack_col[pd_cap_row_idx, 0] = -1.0 - A_ub = hstack([A_ub, csr_matrix(slack_col)], format="csr") - c = np.concatenate([c, np.array([float(pd_cap_slack_penalty)], dtype=float)]) - bounds.append((0.0, float(total_budget))) options: dict[str, Any] = { "time_limit": float(time_limit), @@ -398,10 +473,10 @@ def solve_portfolio_highs_sparse( } _ = threads # SciPy's HiGHS wrapper does not expose a documented threads option. result = linprog( - c, - A_ub=A_ub, - b_ub=np.asarray(rhs, dtype=float), - bounds=bounds, + -components.objective_coefficients.astype(float), + A_ub=components.a_ub, + b_ub=components.rhs, + bounds=components.bounds, method="highs", options=options, ) @@ -412,33 +487,23 @@ def solve_portfolio_highs_sparse( ) primal = np.asarray(result.x, dtype=float) - alloc = np.clip(primal[:n], 0.0, 1.0) - pd_cap_slack = float(primal[-1]) if use_pd_slack else 0.0 - total_allocated = float(np.sum(alloc * loan_amounts)) - obj_value = float(np.sum(alloc * objective) - float(pd_cap_slack_penalty) * pd_cap_slack) - n_funded = int(np.sum(alloc > 0.01)) - allocation = {i: float(value) for i, value in enumerate(alloc) if value > 1e-12} + summary = _portfolio_solution_summary(primal, components) solver_status = "optimal" if bool(result.success) else str(result.message) logger.info( "Portfolio solved (highs_sparse): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, " "pd_cap_slack={:.4f}, status={}", - obj_value, - n_funded, - n, - total_allocated, - pd_cap_slack, + summary["objective_value"], + summary["n_funded"], + components.n, + summary["total_allocated"], + summary["pd_cap_slack"], solver_status, ) return { - "allocation": allocation, - "allocation_vector": alloc, - "objective_value": obj_value, - "n_funded": n_funded, - "total_allocated": total_allocated, + **summary, "solver_status": solver_status, "solver_backend": "highs_sparse", - "pd_cap_slack": pd_cap_slack, "highs_status": int(result.status), "highs_message": str(result.message), "highs_iterations": int(getattr(result, "nit", 0) or 0), @@ -472,21 +537,177 @@ def solve_portfolio_highspy_native( HiGHS' own thread/parallel options while preserving the same LP algebra. """ try: - import highspy # type: ignore[import-not-found] + import highspy except Exception as exc: # pragma: no cover - optional runtime dependency raise RuntimeError( "solver_backend='highspy' requested but highspy is unavailable." ) from exc + components = _portfolio_lp_components( + loans=loans, + pd_point=pd_point, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=max_portfolio_pd, + robust=robust, + uncertainty_aversion=uncertainty_aversion, + min_budget_utilization=min_budget_utilization, + pd_cap_slack_penalty=pd_cap_slack_penalty, + pd_constraint_override=pd_constraint_override, + ) + lp = _build_highspy_lp(highspy, components) + solver = highspy.Highs() + _configure_highspy_solver(highspy, solver, time_limit=time_limit, threads=threads) + + status = solver.passModel(lp) + if status != highspy.HighsStatus.kOk: + raise RuntimeError(f"highspy failed to accept portfolio LP: {status}") + run_status = solver.run() + if run_status == highspy.HighsStatus.kError: + raise RuntimeError(f"highspy failed while solving portfolio LP: {run_status}") + + model_status = solver.getModelStatus() + status_text = str(solver.modelStatusToString(model_status)) + if "Optimal" not in status_text: + raise RuntimeError( + "highspy did not solve portfolio LP to optimality: " + f"run_status={run_status}, model_status={status_text}" + ) + + solution = solver.getSolution() + primal = np.asarray(solution.col_value, dtype=float) + if len(primal) < components.n: + raise RuntimeError( + f"highspy primal solution has length {len(primal)}; expected >= {components.n}." + ) + summary = _portfolio_solution_summary(primal, components) + info = solver.getInfo() + + logger.info( + "Portfolio solved (highspy): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, " + "pd_cap_slack={:.4f}, status={}", + summary["objective_value"], + summary["n_funded"], + components.n, + summary["total_allocated"], + summary["pd_cap_slack"], + status_text, + ) + return { + **summary, + "solver_status": status_text, + "solver_backend": "highspy", + "highs_model_status": status_text, + "highs_simplex_iterations": int(getattr(info, "simplex_iteration_count", 0) or 0), + "highs_ipm_iterations": int(getattr(info, "ipm_iteration_count", 0) or 0), + } + + +def _build_highspy_lp(highspy: Any, components: _PortfolioLpComponents) -> Any: + a_ub: csc_matrix = components.a_ub.tocsc() + bounds = np.asarray(components.bounds, dtype=np.double) + lp = highspy.HighsLp() + lp.num_col_ = int(a_ub.shape[1]) + lp.num_row_ = int(a_ub.shape[0]) + lp.col_cost_ = components.objective_coefficients.astype(np.double).tolist() + lp.col_lower_ = bounds[:, 0].tolist() + lp.col_upper_ = bounds[:, 1].tolist() + lp.row_lower_ = np.full(a_ub.shape[0], -highspy.kHighsInf, dtype=np.double).tolist() + lp.row_upper_ = components.rhs.astype(np.double).tolist() + lp.sense_ = highspy.ObjSense.kMaximize + lp.a_matrix_.format_ = highspy.MatrixFormat.kColwise + lp.a_matrix_.num_col_ = int(a_ub.shape[1]) + lp.a_matrix_.num_row_ = int(a_ub.shape[0]) + lp.a_matrix_.start_ = a_ub.indptr.astype(np.int32).tolist() + lp.a_matrix_.index_ = a_ub.indices.astype(np.int32).tolist() + lp.a_matrix_.value_ = a_ub.data.astype(np.double).tolist() + return lp + + +def _configure_highspy_solver( + highspy: Any, + solver: Any, + *, + time_limit: int, + threads: int, +) -> None: + if _env_int("HIGHS_RESET_GLOBAL_SCHEDULER", 1) and hasattr(solver, "resetGlobalScheduler"): + solver.resetGlobalScheduler(True) + options: dict[str, bool | int | float | str] = { + "output_flag": False, + "log_to_console": False, + "time_limit": float(time_limit), + "presolve": _env_str("HIGHS_PRESOLVE", "on"), + "threads": max(1, int(threads)), + "parallel": _env_str("HIGHS_PARALLEL", "choose"), + "solver": _env_str("HIGHS_SOLVER", "choose"), + "simplex_strategy": _env_int("HIGHS_SIMPLEX_STRATEGY", 0), + } + for name, value in options.items(): + status = _set_highs_option(solver, name, value) + if status != highspy.HighsStatus.kOk and name == "threads": + status = _retry_highspy_threads_option(solver, name, value) + if status != highspy.HighsStatus.kOk: + logger.warning("HiGHS rejected option {}={!r}: {}", name, value, status) + + +def _set_highs_option( + highs_solver: Any, + name: str, + value: bool | int | float | str, +) -> Any: + return highs_solver.setOptionValue(name, value) + + +def _retry_highspy_threads_option( + solver: Any, + name: str, + value: bool | int | float | str, +) -> Any: + if hasattr(solver, "resetGlobalScheduler"): + solver.resetGlobalScheduler(True) + return _set_highs_option(solver, name, value) + + +def _env_str(name: str, default: str) -> str: + return str(os.environ.get(name, default)).strip() or default + + +def _env_int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or not str(raw).strip(): + return int(default) + try: + return int(raw) + except ValueError: + logger.warning("Ignoring invalid {}={!r}; using {}", name, raw, default) + return int(default) + + +def _portfolio_lp_components( + *, + loans: pd.DataFrame, + pd_point: np.ndarray, + pd_high: np.ndarray, + lgd: np.ndarray, + int_rates: np.ndarray, + total_budget: float, + max_concentration: float, + max_portfolio_pd: float, + robust: bool, + uncertainty_aversion: float, + min_budget_utilization: float, + pd_cap_slack_penalty: float, + pd_constraint_override: np.ndarray | None, +) -> _PortfolioLpComponents: n = len(loans) if n == 0: raise ValueError("Cannot solve empty portfolio.") - loan_amounts = ( - pd.to_numeric(loans["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.ones(n, dtype=float) - ) + loan_amounts = _loan_amounts(loans, n) point = np.asarray(pd_point, dtype=float) high = np.asarray(pd_high, dtype=float) lgd_arr = np.asarray(lgd, dtype=float) @@ -497,17 +718,64 @@ def solve_portfolio_highspy_native( else (high if robust else point) ) pd_uncertainty = np.clip(high - point, 0.0, 1.0) - objective = loan_amounts * ( rates - point * lgd_arr - float(uncertainty_aversion) * pd_uncertainty * lgd_arr ) + + rows, rhs, pd_cap_row_idx = _portfolio_constraint_rows( + loans=loans, + loan_amounts=loan_amounts, + pd_constraint=pd_constraint, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=max_portfolio_pd, + min_budget_utilization=min_budget_utilization, + ) + a_ub = csr_matrix(np.vstack(rows).astype(float)) + objective_coefficients = objective.astype(float) + bounds = [(0.0, 1.0)] * n use_pd_slack = float(pd_cap_slack_penalty) > 0.0 + if use_pd_slack: + slack_col = np.zeros((a_ub.shape[0], 1), dtype=float) + slack_col[pd_cap_row_idx, 0] = -1.0 + a_ub = hstack([a_ub, csr_matrix(slack_col)], format="csr") + objective_coefficients = np.concatenate( + [objective_coefficients, np.array([-float(pd_cap_slack_penalty)], dtype=float)] + ) + bounds.append((0.0, float(total_budget))) - rows: list[np.ndarray] = [] - rhs: list[float] = [] + return _PortfolioLpComponents( + n=n, + loan_amounts=loan_amounts, + objective_coefficients=objective_coefficients, + a_ub=a_ub, + rhs=np.asarray(rhs, dtype=float), + bounds=bounds, + use_pd_slack=use_pd_slack, + ) + + +def _loan_amounts(loans: pd.DataFrame, n: int) -> np.ndarray: + if "loan_amnt" not in loans.columns: + return np.ones(n, dtype=float) + return cast( + np.ndarray, + pd.to_numeric(loans["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float), + ) - rows.append(loan_amounts.astype(float)) - rhs.append(float(total_budget)) + +def _portfolio_constraint_rows( + *, + loans: pd.DataFrame, + loan_amounts: np.ndarray, + pd_constraint: np.ndarray, + total_budget: float, + max_concentration: float, + max_portfolio_pd: float, + min_budget_utilization: float, +) -> tuple[list[np.ndarray], list[float], int]: + rows: list[np.ndarray] = [loan_amounts.astype(float)] + rhs: list[float] = [float(total_budget)] min_budget_utilization = float(np.clip(min_budget_utilization, 0.0, 1.0)) if min_budget_utilization > 0: @@ -524,124 +792,24 @@ def solve_portfolio_highspy_native( mask = (purposes == purpose).to_numpy(dtype=float) rows.append((loan_amounts * (mask - float(max_concentration))).astype(float)) rhs.append(0.0) + return rows, rhs, pd_cap_row_idx - A_ub: csc_matrix = csr_matrix(np.vstack(rows).astype(float)).tocsc() - col_cost = objective.astype(np.double) - col_lower = np.zeros(n + int(use_pd_slack), dtype=np.double) - col_upper = np.ones(n + int(use_pd_slack), dtype=np.double) - if use_pd_slack: - slack_col = np.zeros((A_ub.shape[0], 1), dtype=float) - slack_col[pd_cap_row_idx, 0] = -1.0 - A_ub = hstack([A_ub, csc_matrix(slack_col)], format="csc") - col_cost = np.concatenate( - [col_cost, np.array([-float(pd_cap_slack_penalty)], dtype=np.double)] - ) - col_upper[-1] = float(total_budget) - inf = highspy.kHighsInf - lp = highspy.HighsLp() - lp.num_col_ = int(A_ub.shape[1]) - lp.num_row_ = int(A_ub.shape[0]) - lp.col_cost_ = col_cost - lp.col_lower_ = col_lower - lp.col_upper_ = col_upper - lp.row_lower_ = np.full(A_ub.shape[0], -inf, dtype=np.double) - lp.row_upper_ = np.asarray(rhs, dtype=np.double) - lp.sense_ = highspy.ObjSense.kMaximize - lp.a_matrix_.format_ = highspy.MatrixFormat.kColwise - lp.a_matrix_.num_col_ = int(A_ub.shape[1]) - lp.a_matrix_.num_row_ = int(A_ub.shape[0]) - lp.a_matrix_.start_ = A_ub.indptr.astype(np.int32) - lp.a_matrix_.index_ = A_ub.indices.astype(np.int32) - lp.a_matrix_.value_ = A_ub.data.astype(np.double) - - def _env_str(name: str, default: str) -> str: - return str(os.environ.get(name, default)).strip() or default - - def _env_int(name: str, default: int) -> int: - raw = os.environ.get(name) - if raw is None or not str(raw).strip(): - return int(default) - try: - return int(raw) - except ValueError: - logger.warning("Ignoring invalid {}={!r}; using {}", name, raw, default) - return int(default) - - solver = highspy.Highs() - if _env_int("HIGHS_RESET_GLOBAL_SCHEDULER", 1) and hasattr(solver, "resetGlobalScheduler"): - solver.resetGlobalScheduler(True) - options: dict[str, object] = { - "output_flag": False, - "log_to_console": False, - "time_limit": float(time_limit), - "presolve": _env_str("HIGHS_PRESOLVE", "on"), - "threads": max(1, int(threads)), - "parallel": _env_str("HIGHS_PARALLEL", "choose"), - "solver": _env_str("HIGHS_SOLVER", "choose"), - "simplex_strategy": _env_int("HIGHS_SIMPLEX_STRATEGY", 0), - } - for name, value in options.items(): - status = solver.setOptionValue(name, value) - if ( - status != highspy.HighsStatus.kOk - and name == "threads" - and hasattr(solver, "resetGlobalScheduler") - ): - solver.resetGlobalScheduler(True) - status = solver.setOptionValue(name, value) - if status != highspy.HighsStatus.kOk: - logger.warning("HiGHS rejected option {}={!r}: {}", name, value, status) - - status = solver.passModel(lp) - if status != highspy.HighsStatus.kOk: - raise RuntimeError(f"highspy failed to accept portfolio LP: {status}") - run_status = solver.run() - if run_status == highspy.HighsStatus.kError: - raise RuntimeError(f"highspy failed while solving portfolio LP: {run_status}") - - model_status = solver.getModelStatus() - status_text = str(solver.modelStatusToString(model_status)) - if "Optimal" not in status_text: - raise RuntimeError( - "highspy did not solve portfolio LP to optimality: " - f"run_status={run_status}, model_status={status_text}" - ) - - solution = solver.getSolution() - primal = np.asarray(solution.col_value, dtype=float) - if len(primal) < n: - raise RuntimeError(f"highspy primal solution has length {len(primal)}; expected >= {n}.") - alloc = np.clip(primal[:n], 0.0, 1.0) - pd_cap_slack = float(primal[-1]) if use_pd_slack else 0.0 - total_allocated = float(np.sum(alloc * loan_amounts)) - obj_value = float(np.sum(alloc * objective) - float(pd_cap_slack_penalty) * pd_cap_slack) - n_funded = int(np.sum(alloc > 0.01)) - allocation = {i: float(value) for i, value in enumerate(alloc) if value > 1e-12} - info = solver.getInfo() - - logger.info( - "Portfolio solved (highspy): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, " - "pd_cap_slack={:.4f}, status={}", - obj_value, - n_funded, - n, - total_allocated, - pd_cap_slack, - status_text, - ) +def _portfolio_solution_summary( + primal: np.ndarray, + components: _PortfolioLpComponents, +) -> dict[str, Any]: + alloc = np.clip(primal[: components.n], 0.0, 1.0) + pd_cap_slack = float(primal[-1]) if components.use_pd_slack else 0.0 + total_allocated = float(np.sum(alloc * components.loan_amounts)) + objective_value = float(np.dot(primal, components.objective_coefficients)) return { - "allocation": allocation, + "allocation": {i: float(value) for i, value in enumerate(alloc) if value > 1e-12}, "allocation_vector": alloc, - "objective_value": obj_value, - "n_funded": n_funded, + "objective_value": objective_value, + "n_funded": int(np.sum(alloc > 0.01)), "total_allocated": total_allocated, - "solver_status": status_text, - "solver_backend": "highspy", "pd_cap_slack": pd_cap_slack, - "highs_model_status": status_text, - "highs_simplex_iterations": int(getattr(info, "simplex_iteration_count", 0) or 0), - "highs_ipm_iterations": int(getattr(info, "ipm_iteration_count", 0) or 0), } diff --git a/src/utils/mlflow_tracing.py b/src/utils/mlflow_tracing.py index 980bc40..18db545 100644 --- a/src/utils/mlflow_tracing.py +++ b/src/utils/mlflow_tracing.py @@ -102,7 +102,8 @@ def decorator(fn: T) -> T: if not _HAS_MLFLOW or not hasattr(mlflow, "trace"): return fn try: # pragma: no cover — depends on installed MLflow version. - wrapped = mlflow.trace(name=name or fn.__name__)(fn) + span_name = name or getattr(fn, "__name__", "trace") + wrapped = mlflow.trace(name=span_name)(fn) return cast(T, wrapped) except Exception: return fn diff --git a/src/utils/script_helpers.py b/src/utils/script_helpers.py index c381c01..72656e9 100644 --- a/src/utils/script_helpers.py +++ b/src/utils/script_helpers.py @@ -76,6 +76,8 @@ def write_table( match, so frozen tables under ``reports/crpto/tables`` keep their manifest hashes when regenerated from unchanged inputs. """ + table_dir = table_dir.resolve() + root = root.resolve() table_dir.mkdir(parents=True, exist_ok=True) csv_path = table_dir / f"{name}.csv" tex_path = table_dir / f"{name}.tex" @@ -101,6 +103,35 @@ def artifact_path(path_like: str | Path) -> Path: return (Path(root) / path) if root else path +def resolve_repo_artifact_path( + path_like: str | Path, + *, + root: Path = REPO_ROOT, +) -> Path: + """Resolve relative or foreign-OS paths that point inside this repository. + + Experiment manifests may have been written from WSL and therefore contain + ``/mnt/c/...//...`` paths. The artifact identity is repository-relative, + so a native Windows replay should resolve the suffix under its current root. + Absolute paths outside this repository are left unchanged. + """ + path = Path(path_like) + if path.exists(): + return path.resolve() + + normalized_parts = [part for part in str(path_like).replace("\\", "/").split("/") if part] + root_name = root.name.casefold() + matching_indices = [ + index for index, part in enumerate(normalized_parts) if part.casefold() == root_name + ] + if matching_indices: + suffix = normalized_parts[matching_indices[-1] + 1 :] + return root.joinpath(*suffix) + if not path.is_absolute(): + return root / path + return path + + def first_existing(*paths: Path) -> Path: """Return the first existing path, falling back to the last candidate.""" if not paths: diff --git a/tests/test_crpto_final_sync.py b/tests/test_crpto_final_sync.py index d240d69..254194c 100644 --- a/tests/test_crpto_final_sync.py +++ b/tests/test_crpto_final_sync.py @@ -32,9 +32,8 @@ def load_key_metrics() -> dict[str, str]: def test_crpto_champion_artifacts_agree() -> None: """Validate the historical frozen rebaseline chain (two-tag scheme). - Under the pool93 promotion this run tag remains the frozen upstream - chain and declared return floor; the paper body claim is guarded by - tests/test_pool93_body_claim_sync.py. + This run tag remains the frozen upstream provenance chain. The active + manuscript policy is guarded by tests/test_ijds_active_claim_sync.py. """ assert Path("data/processed/final_project_summary.parquet.dvc").exists() promotion = load_json("models/final_project_promotion.json") @@ -98,11 +97,12 @@ def test_crpto_tables_and_figures_exist() -> None: "reports/crpto/tables/crpto_tableA22_tail_constrained_reoptimization.csv", "reports/crpto/tables/crpto_tableA23_multidistribution_robustness.csv", "reports/crpto/tables/crpto_tableA24_online_conformal_stability.csv", - "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv", - "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", - "reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv", - "reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv", - "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv", + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", "reports/crpto/figures/crpto_fig1_journal_pipeline.png", "reports/crpto/figures/crpto_fig1_journal_pipeline.pdf", "reports/crpto/figures/crpto_fig1_journal_pipeline.svg", @@ -110,8 +110,6 @@ def test_crpto_tables_and_figures_exist() -> None: "book/assets/figures/publication/crpto_fig1_journal_pipeline.pdf", "book/assets/figures/publication/crpto_fig1_journal_pipeline.svg", "reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.png", - "reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.png", - "reports/crpto/figures/crpto_fig14_robust_region_heatmap.png", "reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png", "reports/crpto/figures/crpto_fig20_bound_claim_layers.png", ] diff --git a/tests/test_evaluation/test_model_shift.py b/tests/test_evaluation/test_model_shift.py index 8454e31..ba18f8a 100644 --- a/tests/test_evaluation/test_model_shift.py +++ b/tests/test_evaluation/test_model_shift.py @@ -39,3 +39,42 @@ def test_interpret_model_shift_detects_mixed_shift() -> None: assert out["shift_type"] == "mixed_shift" assert out["governance_posture"] == "candidate_gate" + + +def test_interpret_model_shift_detects_predictive_degradation_without_structural_shift() -> None: + out = interpret_model_shift( + c2st_auc=0.50, + c2st_materiality="none", + score_psi=0.02, + auc_delta=0.08, + brier_increase=0.001, + calibration_gap_delta=0.001, + distribution_warning_ratio=0.0, + score_psi_max=0.15, + auc_delta_max=0.05, + brier_increase_max=0.02, + calibration_gap_delta_max=0.02, + ) + + assert out["shift_type"] == "predictive_degradation" + assert out["governance_posture"] == "candidate_gate" + assert "Operational metrics dominate" in str(out["pvalue_interpretation"]) + + +def test_interpret_model_shift_keeps_stable_cases_on_monitor_posture() -> None: + out = interpret_model_shift( + c2st_auc=0.50, + c2st_materiality="none", + score_psi=0.01, + auc_delta=0.0, + brier_increase=0.0, + calibration_gap_delta=0.0, + distribution_warning_ratio=0.0, + score_psi_max=0.15, + auc_delta_max=0.05, + brier_increase_max=0.02, + calibration_gap_delta_max=0.02, + ) + + assert out["shift_type"] == "stable" + assert out["governance_posture"] == "monitor" diff --git a/tests/test_experiments/test_champion_reopen_orchestration.py b/tests/test_experiments/test_champion_reopen_orchestration.py index 84e1345..5f7dca4 100644 --- a/tests/test_experiments/test_champion_reopen_orchestration.py +++ b/tests/test_experiments/test_champion_reopen_orchestration.py @@ -123,6 +123,7 @@ def test_portfolio_command_separates_proxy_and_exact_sampling(tmp_path) -> None: "solver_backend": "cuopt", "exact_solver_backend": "highs", "frontier_only": True, + "exact_python_executable": "python-exact", }, "frontier": { "proxy_candidates_per_conformal_finalist": 100000, @@ -147,6 +148,11 @@ def test_portfolio_command_separates_proxy_and_exact_sampling(tmp_path) -> None: "gamma_neighbors": "0.45", "policy_modes": "blended_uncertainty", }, + "cuopt": { + "method": "concurrent", + "num_gpus": 1, + "extra_parameters": {"tolerance": "tight"}, + }, }, conformal_intervals_path=tmp_path / "intervals.parquet", run_label="unit", @@ -160,4 +166,8 @@ def test_portfolio_command_separates_proxy_and_exact_sampling(tmp_path) -> None: assert command[command.index("--exact-checkpoint-every") + 1] == "25" assert command[command.index("--exact-threads") + 1] == "8" assert command[command.index("--budget-profiles") + 1] == "free" + assert command[command.index("--exact-python-executable") + 1] == "python-exact" + assert command[command.index("--cuopt-method") + 1] == "concurrent" + assert command[command.index("--cuopt-num-gpus") + 1] == "1" + assert "tolerance=tight" in command assert "--frontier-only" in command diff --git a/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py b/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py new file mode 100644 index 0000000..db4dd2d --- /dev/null +++ b/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd + +from scripts.experiments.run_ijds_calibration_selected_policy_challenger import ( + FORBIDDEN_SELECTOR_COLUMNS, + _funded_allocation_frame, + _measure_ex_ante_solution, + _selector_input_frame, +) +from src.optimization.policy import PolicyMode +from src.optimization.policy_evaluation import PolicyAllocationResult +from src.optimization.policy_selection import LinearPolicyCandidate + + +def test_ex_ante_measurement_contains_no_outcome_fields() -> None: + frame = pd.DataFrame( + { + "_loan_amount": [100.0, 200.0], + "_pd_point": [0.10, 0.20], + "_pd_high": [0.30, 0.50], + } + ) + candidate = LinearPolicyCandidate( + candidate_id="linear-001", + risk_tolerance=0.20, + gamma=0.50, + uncertainty_aversion=0.0, + ) + result = PolicyAllocationResult( + solution={ + "solver_status": "Optimal", + "objective_value": 10.0, + }, + allocation=np.array([1.0, 0.5]), + effective_pd=np.array([0.20, 0.35]), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.50, + delta_cap_quantile=1.0, + tail_focus_quantile=1.0, + objective_risk_mode="point_pd_plus_aversion", + ) + + record = _measure_ex_ante_solution(frame, candidate, result, alpha=0.10) + + assert not FORBIDDEN_SELECTOR_COLUMNS.intersection(record) + assert record["expected_objective"] == 10.0 + assert record["weighted_pd_effective"] == 0.275 + + +def test_selector_input_frame_drops_all_outcomes() -> None: + frame = pd.DataFrame( + { + "id": ["a"], + "loan_amnt": [100.0], + "purpose": ["credit_card"], + "issue_d": ["2017-11-01"], + "default_flag": [1], + "y_true": [1], + "loan_status": ["Charged Off"], + "_outcome": [1.0], + "_pd_point": [0.1], + "_pd_low": [0.0], + "_pd_high": [0.3], + "_loan_amount": [100.0], + "_int_rate": [0.2], + } + ) + + selector = _selector_input_frame(frame) + + assert not {"default_flag", "y_true", "loan_status", "_outcome"}.intersection(selector.columns) + + +def test_funded_allocation_frame_reconciles_exposure_and_return() -> None: + frame = pd.DataFrame( + { + "id": ["a", "b"], + "issue_d": ["2020-01-01", "2020-02-01"], + "grade": ["A", "B"], + "_loan_amount": [100.0, 200.0], + "_int_rate": [0.10, 0.20], + "_outcome": [0.0, 1.0], + "_pd_point": [0.10, 0.20], + "_pd_low": [0.0, 0.0], + "_pd_high": [0.30, 0.50], + } + ) + result = PolicyAllocationResult( + solution={"solver_status": "Optimal", "objective_value": 10.0}, + allocation=np.array([1.0, 0.5]), + effective_pd=np.array([0.20, 0.35]), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.50, + delta_cap_quantile=1.0, + tail_focus_quantile=1.0, + objective_risk_mode="point_pd_plus_aversion", + ) + + funded = _funded_allocation_frame(frame, result, role="selected", lgd=0.45) + + assert funded["funded_exposure"].sum() == 200.0 + assert funded["funded_weight"].sum() == 1.0 + assert funded["realized_return_contribution"].sum() == -35.0 diff --git a/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py b/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py new file mode 100644 index 0000000..4ab34c0 --- /dev/null +++ b/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd + +from scripts.experiments.run_ijds_exact_alpha_grid_challenger import ( + _base_grid_frame, + _replay_differences, +) +from src.models.conformal_alpha_grid import ExactAlphaIntervals + + +def _result() -> ExactAlphaIntervals: + return ExactAlphaIntervals( + target_alpha=0.10, + used_alpha=0.095, + point=np.array([0.2, 0.4]), + low=np.array([0.0, 0.1]), + high=np.array([0.5, 0.8]), + partition_labels=pd.Series(["a", "b"]), + partition_metadata={}, + diagnostics={}, + ) + + +def test_base_grid_keeps_only_traceability_columns() -> None: + source = pd.DataFrame( + { + "_row_number": [0], + "id": ["x"], + "y_true": [0.0], + "grade": ["A"], + "pd_high_90": [0.5], + } + ) + + assert _base_grid_frame(source).columns.tolist() == [ + "_row_number", + "id", + "y_true", + "grade", + ] + + +def test_reference_replay_reports_exact_match() -> None: + source = pd.DataFrame( + { + "y_pred": [0.2, 0.4], + "pd_low_90": [0.0, 0.1], + "pd_high_90": [0.5, 0.8], + } + ) + + assert _replay_differences(_result(), source) == { + "point_max_abs": 0.0, + "low_max_abs": 0.0, + "high_max_abs": 0.0, + } diff --git a/tests/test_ijds_active_claim_sync.py b/tests/test_ijds_active_claim_sync.py new file mode 100644 index 0000000..29c95c0 --- /dev/null +++ b/tests/test_ijds_active_claim_sync.py @@ -0,0 +1,153 @@ +"""Drift guard for the active calibration-selected IJDS policy.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +import pytest + +REPO = Path(__file__).resolve().parents[1] +TABLES = REPO / "reports/crpto/tables" +RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" +GOVERNANCE = ( + REPO / "models/experiments/champion_reopen" / RUN_TAG / "portfolio/ijds_policy_governance.json" +) +SUMMARY = GOVERNANCE.with_name("calibration_selected_policy_summary.json") +SURFACES = ( + REPO / "paper/CRPTO_ijds.qmd", + REPO / "paper/supplement_ijds.qmd", + REPO / "paper/submission/CRPTO_ijds_submission.tex", +) +TABLE_STEMS = ( + "crpto_tableA35_exact_alpha_grid", + "crpto_tableA36_calibration_policy_selector", + "crpto_tableA37_calibration_selected_temporal_evaluation", + "crpto_tableA38_calibration_selected_grade_audit", + "crpto_tableA39_calibration_selected_bootstrap", + "crpto_tableA40_calibration_selected_point_baseline", +) + + +def _json(path: Path) -> dict[str, Any]: + assert path.is_file(), f"Missing active governance: {path}" + return json.loads(path.read_text(encoding="utf-8")) + + +def _rows(stem: str) -> list[dict[str, str]]: + with (TABLES / f"{stem}.csv").open(encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def _surface_text(path: Path) -> str: + return path.read_text(encoding="utf-8").replace(r"\$", "$").replace("{,}", ",") + + +def test_active_governance_locks_simple_policy_and_selector() -> None: + payload = _json(GOVERNANCE) + summary = _json(SUMMARY) + policy = payload["selected_policy"] + selector = payload["selection_protocol"] + + assert payload["status"] == "active_ijds_policy" + assert payload["run_tag"] == RUN_TAG + assert payload["generated_at_utc"] == summary["generated_at_utc"] + assert policy["policy_mode"] == "blended_uncertainty" + assert policy["risk_tolerance"] == pytest.approx(0.17) + assert policy["gamma"] == pytest.approx(0.50) + assert policy["uncertainty_aversion"] == pytest.approx(0.0) + assert policy["min_budget_utilization"] == pytest.approx(0.0) + assert selector["min_budget_utilization"] == pytest.approx(0.999) + assert selector["n_total"] == 9 + assert selector["n_eligible"] == 5 + assert selector["outcome_columns_used"] == 0 + assert selector["statistical_assumption_columns_used"] == 0 + assert selector["endpoint_budget_cap"] == pytest.approx(0.28) + assert selector["selector_forbidden_columns_present"] == [] + assert selector["calibration_metadata"]["target_alpha"] == pytest.approx(0.10) + assert selector["calibration_metadata"]["selection_period"] == "2017-11" + assert selector["calibration_metadata"]["selection_rows"] == 14943 + assert selector["calibration_metadata"]["audit_period"] == "2017-12" + assert selector["calibration_metadata"]["audit_rows"] == 20695 + assert selector["calibration_metadata"]["outcomes_isolated_until_post_selection_audit"] is True + assert "_outcome" not in selector["selector_input_columns"] + assert selector["endpoint_cap_stability"]["cap_lower_inclusive"] == pytest.approx( + 0.25903604939435104 + ) + assert selector["endpoint_cap_stability"]["cap_upper_exclusive"] == pytest.approx( + 0.29049078888716334 + ) + assert selector["calibration_audit"]["same_policy_selected"] is True + assert selector["calibration_audit"]["selected_policy"][ + "weighted_miscoverage" + ] == pytest.approx(0.124925) + assert payload["exact_alpha_reference_replay"]["pass"] is True + + +def test_active_tables_agree_with_governance() -> None: + payload = _json(GOVERNANCE) + full = payload["full_oot"] + selected = next( + row + for row in _rows("crpto_tableA40_calibration_selected_point_baseline") + if row["policy"] == "Calibration-selected 50/50 CRPTO" + ) + alpha = next( + row + for row in _rows("crpto_tableA35_exact_alpha_grid") + if row["selected_for_policy"] == "True" + ) + + assert float(selected["realized_return"]) == pytest.approx(full["realized_return"]) + assert float(selected["weighted_outcome"]) == pytest.approx(full["weighted_default_rate"]) + assert float(selected["weighted_miscoverage"]) == pytest.approx(full["weighted_miscoverage"]) + assert float(selected["endpoint_budget"]) == pytest.approx(full["endpoint_budget"]) + assert float(selected["markov_loss_threshold"]) == pytest.approx(full["markov_loss_threshold"]) + assert float(alpha["target_alpha"]) == pytest.approx(0.10) + assert float(alpha["empirical_coverage"]) == pytest.approx(0.9348356081757077) + + +def test_active_a35_to_a40_exist_in_csv_and_tex() -> None: + missing = [ + f"{stem}.{suffix}" + for stem in TABLE_STEMS + for suffix in ("csv", "tex") + if not (TABLES / f"{stem}.{suffix}").is_file() + ] + assert not missing, "Missing active IJDS evidence: " + ", ".join(missing) + + +def test_active_manuscript_surfaces_share_numeric_anchors() -> None: + payload = _json(GOVERNANCE) + full = payload["full_oot"] + contrast = payload["point_pd_contrast"] + anchors = ( + f"${full['realized_return']:,.2f}", + f"{full['weighted_default_rate']:.6f}", + f"{full['weighted_miscoverage']:.6f}", + f"{full['Gamma_CP']:.6f}", + f"{full['Gamma_residual']:.6f}", + f"{full['endpoint_budget']:.6f}", + f"{full['observed_accounting_bound']:.6f}", + f"{full['markov_loss_threshold']:.6f}", + f"{contrast['realized_return']:,.2f}", + f"{100 * contrast['selected_return_cost_pct']:.3f}", + f"{100 * contrast['selected_default_reduction']:.4f}", + "0.28", + "0.124925", + "$163,421.14", + ) + for surface in SURFACES: + text = _surface_text(surface) + missing = [anchor for anchor in anchors if anchor not in text] + assert not missing, f"{surface.name} missing active anchors: {missing}" + + +def test_retired_headline_numbers_do_not_appear_in_active_surfaces() -> None: + retired = ("0.345084", "50,010", "27,508", "capped_blended_uncertainty") + for surface in SURFACES: + text = _surface_text(surface) + present = [token for token in retired if token in text] + assert not present, f"{surface.name} retains retired claims: {present}" diff --git a/tests/test_manifest_regression.py b/tests/test_manifest_regression.py index 807076c..6cbc9be 100644 --- a/tests/test_manifest_regression.py +++ b/tests/test_manifest_regression.py @@ -50,6 +50,15 @@ "models/experiments/champion_reopen/" "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/" "portfolio/pool93_ijds_consolidated_governance.json", + "models/experiments/champion_reopen/" + "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" + "portfolio/pool93_ijds_consolidated_frontier.json", + "models/experiments/champion_reopen/" + "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" + "portfolio/pool93_ijds_consolidated_governance.json", + "models/experiments/champion_reopen/" + "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" + "portfolio/pool93_point_pd_baseline_audit.json", ) diff --git a/tests/test_models/test_conformal_alpha_grid.py b/tests/test_models/test_conformal_alpha_grid.py new file mode 100644 index 0000000..8864a2e --- /dev/null +++ b/tests/test_models/test_conformal_alpha_grid.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from src.models.conformal_alpha_grid import ( + FrozenConformalRecipe, + alpha_interval_columns, + compute_exact_alpha_intervals, +) + + +def _payload() -> dict: + return { + "tuning_90_best": { + "partition": "grade", + "partition_probability_source": "calibrated", + "n_score_bins": 5, + "fallback_mode": "global_only", + "score_scale_family": "none", + "min_group_size": 1, + "alpha_target_90": 0.10, + "alpha_used_90": 0.095, + }, + "calibration_split": { + "calibration_fraction": 0.75, + "holdout_ratio": 0.20, + "random_state": 42, + }, + "group_coverage_multipliers": {"A": 1.05}, + "temporal_segment_multipliers": {}, + "global_rebalance": {"enabled": False, "applied": False}, + } + + +def test_recipe_scales_alpha_by_frozen_conservative_ratio() -> None: + recipe = FrozenConformalRecipe.from_results_payload(_payload()) + + assert recipe.used_alpha(0.01) == pytest.approx(0.0095) + assert alpha_interval_columns(0.01) == ( + "pd_low_alpha_0p010", + "pd_high_alpha_0p010", + ) + + +def test_recipe_rejects_narrowing_adjustments() -> None: + payload = _payload() + payload["global_rebalance"] = {"applied": True, "factor": 0.95} + + with pytest.raises(ValueError, match="widening"): + FrozenConformalRecipe.from_results_payload(payload) + + +def test_exact_alpha_intervals_use_frozen_partition_and_multiplier() -> None: + recipe = FrozenConformalRecipe.from_results_payload(_payload()) + y_cal = np.array([0.0, 1.0, 0.0, 1.0]) + p_cal = np.array([0.1, 0.8, 0.2, 0.7]) + p_eval = np.array([0.15, 0.75]) + groups_cal = pd.Series(["A", "A", "B", "B"]) + groups_eval = pd.Series(["A", "B"]) + + result = compute_exact_alpha_intervals( + recipe=recipe, + target_alpha=0.10, + y_cal=y_cal, + interval_probability_cal=p_cal, + interval_probability_eval=p_eval, + partition_probability_cal=p_cal, + partition_probability_eval=p_eval, + base_groups_cal=groups_cal, + base_groups_eval=groups_eval, + ) + + assert result.used_alpha == pytest.approx(0.095) + assert result.partition_labels.tolist() == ["A", "B"] + assert np.all(result.low <= result.point) + assert np.all(result.high >= result.point) + assert result.high[0] == pytest.approx(0.36) diff --git a/tests/test_models/test_conformal_tuning.py b/tests/test_models/test_conformal_tuning.py index 08f1a78..8ae730e 100644 --- a/tests/test_models/test_conformal_tuning.py +++ b/tests/test_models/test_conformal_tuning.py @@ -385,6 +385,65 @@ def test_reduces_width_while_preserving_constraints(self): assert all(f <= 1.20 for f in shrunk_group_factors.values()) assert shrunk_temporal_factors == {} + def test_temporal_factors_shrink_when_base_intervals_are_feasible(self): + n = 48 + y_pred = np.full(n, 0.50) + y_true = np.full(n, 0.50) + base_intervals = np.column_stack([np.full(n, 0.45), np.full(n, 0.55)]) + groups = np.array(["A"] * n) + temporal_segments = np.array(["early"] * (n // 2) + ["late"] * (n // 2)) + issue_dates = pd.Series(pd.date_range("2024-01-01", periods=n, freq="D")) + + widened, group_factors, temporal_factors, report = shrink_group_multipliers( + y_true=y_true, + y_pred=y_pred, + base_intervals=base_intervals, + groups=groups, + issue_dates=issue_dates, + group_factors=None, + temporal_segments=temporal_segments, + temporal_factors={"early": 1.20, "late": 1.20}, + target_coverage=0.90, + min_group_coverage_target=0.88, + temporal_multiplier_grid=(1.0, 1.20), + ) + + accepted_temporal = report[ + (report["stage"] == "accepted") & (report["factor_scope"] == "temporal") + ] + + assert np.allclose(widened, base_intervals) + assert group_factors == {} + assert temporal_factors == {} + assert set(accepted_temporal["factor_key"]) == {"early", "late"} + + def test_returns_initial_infeasible_report_without_shrinking(self): + n = 20 + y_pred = np.full(n, 0.50) + y_true = np.full(n, 0.95) + base_intervals = np.column_stack([np.full(n, 0.45), np.full(n, 0.55)]) + groups = np.array(["A"] * (n // 2) + ["B"] * (n // 2)) + issue_dates = pd.Series(pd.date_range("2024-01-01", periods=n, freq="D")) + + _widened, group_factors, temporal_factors, report = shrink_group_multipliers( + y_true=y_true, + y_pred=y_pred, + base_intervals=base_intervals, + groups=groups, + issue_dates=issue_dates, + group_factors={"A": 1.20}, + temporal_segments=None, + temporal_factors=None, + target_coverage=0.90, + min_group_coverage_target=0.88, + group_multiplier_grid=(1.0, 1.20), + ) + + assert report["stage"].tolist() == ["initial", "initial_infeasible"] + assert report["accepted"].tolist() == [True, False] + assert group_factors == {"A": 1.20} + assert temporal_factors == {} + # --------------------------------------------------------------------------- # to_python_scalar diff --git a/tests/test_optimization/test_certificate_semantics.py b/tests/test_optimization/test_certificate_semantics.py new file mode 100644 index 0000000..3b17f8e --- /dev/null +++ b/tests/test_optimization/test_certificate_semantics.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import yaml + +from src.optimization.certificate_semantics import ( + IJDS_DECLARED_ALPHA_GRID, + IJDS_DECLARED_ALPHA_GRID_CSV, + add_policy_aware_bound_columns, + compute_funded_certificate_metrics, +) + +ROOT = Path(__file__).resolve().parents[2] + + +def test_ijds_alpha_grid_matches_profile_and_claim_registry() -> None: + profile_path = ROOT / "configs" / "profiles" / "search_portfolio_pool93_stage1_claim_26_06.yaml" + profile = yaml.safe_load(profile_path.read_text(encoding="utf-8")) + claims = (ROOT / "docs" / "research" / "active_claims_2026-07-04.md").read_text( + encoding="utf-8" + ) + claim_grid = ", ".join(f"{alpha:.2f}" for alpha in IJDS_DECLARED_ALPHA_GRID) + + assert profile["grids"]["alpha_grid"] == IJDS_DECLARED_ALPHA_GRID_CSV + assert f"`A = {{{claim_grid}}}`" in claims + + +def test_linear_blend_certificate_decomposition() -> None: + weights = np.array([0.25, 0.75]) + point = np.array([0.10, 0.20]) + high = np.array([0.50, 0.60]) + gamma = 0.50 + effective = point + gamma * (high - point) + + metrics = compute_funded_certificate_metrics( + weights, + outcomes=np.array([0.0, 1.0]), + pd_point=point, + pd_high=high, + pd_effective=effective, + alpha=0.01, + risk_tolerance=float(weights @ effective), + ) + + assert metrics.gamma_cp == pytest.approx(0.40) + assert metrics.gamma_internalized == pytest.approx(gamma * metrics.gamma_cp) + assert metrics.gamma_residual == pytest.approx((1.0 - gamma) * metrics.gamma_cp) + assert metrics.endpoint_budget == pytest.approx(float(weights @ high)) + assert metrics.endpoint_budget_upper == pytest.approx(metrics.endpoint_budget) + assert metrics.markov_loss_threshold == pytest.approx(metrics.endpoint_budget + 0.10) + assert metrics.markov_loss_cap == pytest.approx(metrics.markov_loss_threshold) + + +def test_tail_policy_uses_actual_residual_not_linear_blend_shortcut() -> None: + weights = np.array([0.50, 0.50]) + point = np.array([0.10, 0.10]) + high = np.array([0.50, 0.90]) + gamma = 0.50 + effective = np.array([0.10, 0.50]) + + metrics = compute_funded_certificate_metrics( + weights, + outcomes=np.array([0.0, 1.0]), + pd_point=point, + pd_high=high, + pd_effective=effective, + alpha=0.01, + risk_tolerance=float(weights @ effective), + ) + + linear_shortcut = (1.0 - gamma) * metrics.gamma_cp + assert metrics.gamma_cp == pytest.approx(0.60) + assert metrics.gamma_internalized == pytest.approx(0.20) + assert metrics.gamma_residual == pytest.approx(0.40) + assert linear_shortcut == pytest.approx(0.30) + assert metrics.gamma_residual > linear_shortcut + assert metrics.endpoint_budget_upper == pytest.approx(metrics.endpoint_budget) + + +def test_policy_cap_slack_is_included_in_endpoint_upper() -> None: + metrics = compute_funded_certificate_metrics( + weights=np.array([1.0]), + outcomes=np.array([1.0]), + pd_point=np.array([0.10]), + pd_high=np.array([0.60]), + pd_effective=np.array([0.40]), + alpha=0.04, + risk_tolerance=0.35, + pd_cap_slack=0.05, + ) + + assert metrics.endpoint_budget == pytest.approx(0.60) + assert metrics.gamma_residual == pytest.approx(0.20) + assert metrics.endpoint_budget_upper == pytest.approx(0.60) + assert metrics.markov_loss_cap == pytest.approx(0.80) + assert metrics.effective_constraint_excess == pytest.approx(0.0) + + +@pytest.mark.parametrize( + ("weights", "match"), + [ + (np.array([0.25, 0.25]), "sum to one"), + (np.array([1.1, -0.1]), "nonnegative"), + ], +) +def test_certificate_rejects_invalid_weights(weights: np.ndarray, match: str) -> None: + with pytest.raises(ValueError, match=match): + compute_funded_certificate_metrics( + weights=weights, + outcomes=np.array([0.0, 1.0]), + pd_point=np.array([0.10, 0.20]), + pd_high=np.array([0.30, 0.40]), + pd_effective=np.array([0.20, 0.30]), + alpha=0.01, + risk_tolerance=0.30, + ) + + +def test_bound_frame_rehydrates_tail_policy_without_linear_shortcut() -> None: + frame = pd.DataFrame( + { + "alpha": [0.01], + "risk_tolerance": [0.20], + "gamma": [0.50], + "gamma_cp": [0.60], + "weighted_pd_point": [0.10], + "weighted_pd_constraint_used": [0.20], + "weighted_pd_high": [0.60], + "pd_cap_slack": [0.0], + } + ) + + result = add_policy_aware_bound_columns(frame).iloc[0] + + assert result["gamma_internalized"] == pytest.approx(0.10) + assert result["gamma_residual"] == pytest.approx(0.40) + assert result["endpoint_budget"] == pytest.approx(0.60) + assert result["endpoint_budget_upper"] == pytest.approx(0.60) + assert result["markov_loss_threshold"] == pytest.approx(0.70) + assert result["markov_loss_cap"] == pytest.approx(0.70) + assert pytest.approx(0.50) == 0.20 + (1.0 - 0.50) * 0.60 + + +def test_bound_frame_separates_exact_threshold_from_slack_upper() -> None: + frame = pd.DataFrame( + { + "alpha": [0.04], + "tau": [0.50], + "weighted_pd_constraint_used": [0.40], + "weighted_pd_high": [0.60], + } + ) + + result = add_policy_aware_bound_columns(frame).iloc[0] + + assert result["effective_constraint_slack"] == pytest.approx(0.10) + assert result["endpoint_budget"] == pytest.approx(0.60) + assert result["endpoint_budget_upper"] == pytest.approx(0.70) + assert result["markov_loss_threshold"] == pytest.approx(0.80) + assert result["markov_loss_cap"] == pytest.approx(0.90) diff --git a/tests/test_optimization/test_cuopt_adapter.py b/tests/test_optimization/test_cuopt_adapter.py new file mode 100644 index 0000000..70d6d8c --- /dev/null +++ b/tests/test_optimization/test_cuopt_adapter.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd +import pytest + +import src.optimization.cuopt_adapter as cuopt_adapter + + +class _FakeDataModel: + def set_csr_constraint_matrix( + self, + data: np.ndarray, + indices: np.ndarray, + indptr: np.ndarray, + ) -> None: + self.data = data + self.indices = indices + self.indptr = indptr + + def set_constraint_bounds(self, bounds: np.ndarray) -> None: + self.constraint_bounds = bounds + + def set_row_types(self, row_types: np.ndarray) -> None: + self.row_types = row_types + + def set_objective_coefficients(self, objective: np.ndarray) -> None: + self.objective = objective + + def set_maximize(self, maximize: bool) -> None: + self.maximize = maximize + + def set_variable_lower_bounds(self, bounds: np.ndarray) -> None: + self.variable_lower_bounds = bounds + + def set_variable_upper_bounds(self, bounds: np.ndarray) -> None: + self.variable_upper_bounds = bounds + + +class _FakeSettings: + def __init__(self) -> None: + self.parameters: dict[str, Any] = {} + + def set_parameter(self, name: str, value: Any) -> None: + self.parameters[name] = value + + +class _FakeSolution: + def __init__( + self, + primal: np.ndarray, + *, + objective: float = 12.5, + termination_reason: str = "Optimal", + ) -> None: + self._primal = primal + self._objective = objective + self._termination_reason = termination_reason + + def get_primal_solution(self) -> np.ndarray: + return self._primal + + def get_primal_objective(self) -> float: + return self._objective + + def get_termination_reason(self) -> str: + return self._termination_reason + + +class _FakeLpApi: + def __init__(self, solution: _FakeSolution) -> None: + self.solution = solution + self.data_model: _FakeDataModel | None = None + self.settings: _FakeSettings | None = None + + def DataModel(self) -> _FakeDataModel: + self.data_model = _FakeDataModel() + return self.data_model + + def SolverSettings(self) -> _FakeSettings: + self.settings = _FakeSettings() + return self.settings + + def Solve(self, data_model: _FakeDataModel, settings: _FakeSettings) -> _FakeSolution: + self.solved_data_model = data_model + self.solved_settings = settings + return self.solution + + +def test_cuopt_native_uses_shared_lp_components_and_solver_settings( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + loans = pd.DataFrame( + { + "loan_amnt": [100.0, 200.0, 300.0], + "purpose": ["debt", "home", "home"], + } + ) + fake_api = _FakeLpApi(_FakeSolution(np.array([0.50, 0.25, 0.00, 3.00]))) + monkeypatch.setattr(cuopt_adapter, "_require_cuopt", lambda: fake_api) + + result = cuopt_adapter.solve_portfolio_cuopt_native( + loans=loans, + pd_point=np.array([0.02, 0.04, 0.06]), + pd_high=np.array([0.03, 0.05, 0.08]), + lgd=np.array([0.45, 0.45, 0.45]), + int_rates=np.array([0.12, 0.14, 0.16]), + total_budget=1_000.0, + max_concentration=0.75, + max_portfolio_pd=0.10, + robust=True, + min_budget_utilization=0.20, + pd_cap_slack_penalty=0.50, + time_limit=15, + random_seed=9, + presolve=0, + cuopt_parameters={ + "log_to_console": "true", + "log_dir": str(tmp_path), + "num_cpu_threads": "2", + }, + ) + + assert fake_api.data_model is not None + assert fake_api.settings is not None + assert fake_api.data_model.maximize is True + assert fake_api.data_model.constraint_bounds[0] == pytest.approx(1_000.0) + assert fake_api.data_model.variable_lower_bounds.tolist() == [0.0, 0.0, 0.0, 0.0] + assert fake_api.data_model.variable_upper_bounds.tolist() == [1.0, 1.0, 1.0, 1_000.0] + assert set(fake_api.data_model.row_types.tolist()) == {"L"} + assert fake_api.settings.parameters["time_limit"] == 15 + assert fake_api.settings.parameters["random_seed"] == 9 + assert fake_api.settings.parameters["presolve"] == 0 + assert fake_api.settings.parameters["log_to_console"] is True + assert fake_api.settings.parameters["num_cpu_threads"] == 2 + assert str(tmp_path) in str(fake_api.settings.parameters["log_file"]) + + assert result["solver_backend"] == "cuopt" + assert result["allocation"] == {0: 0.5, 1: 0.25, 2: 0.0} + assert np.allclose(result["allocation_vector"], np.array([0.5, 0.25, 0.0])) + assert result["total_allocated"] == pytest.approx(100.0) + assert result["pd_cap_slack"] == pytest.approx(3.0) + assert result["cuopt_log_file"] == fake_api.settings.parameters["log_file"] + + +def test_cuopt_native_rejects_non_feasible_termination( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loans = pd.DataFrame({"loan_amnt": [100.0], "purpose": ["debt"]}) + fake_api = _FakeLpApi(_FakeSolution(np.array([0.0]), termination_reason="Infeasible")) + monkeypatch.setattr(cuopt_adapter, "_require_cuopt", lambda: fake_api) + + with pytest.raises(RuntimeError, match="did not produce an acceptable solution"): + cuopt_adapter.solve_portfolio_cuopt_native( + loans=loans, + pd_point=np.array([0.02]), + pd_high=np.array([0.03]), + lgd=np.array([0.45]), + int_rates=np.array([0.12]), + ) diff --git a/tests/test_optimization/test_input_alignment.py b/tests/test_optimization/test_input_alignment.py new file mode 100644 index 0000000..5616148 --- /dev/null +++ b/tests/test_optimization/test_input_alignment.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from src.optimization.input_alignment import align_candidate_intervals + + +def test_id_alignment_preserves_left_order_and_interval_payload() -> None: + candidates = pd.DataFrame( + { + "id": ["b", "a", "c"], + "grade": ["B", "A", "C"], + "y_pred": [9.0, 8.0, 7.0], + } + ) + intervals = pd.DataFrame( + { + "id": ["a", "b", "c"], + "grade": ["score_q01", "score_q02", "score_q03"], + "y_pred": [0.1, 0.2, 0.3], + } + ) + + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=None, + random_state=42, + ) + + assert aligned.mode == "id" + assert aligned.available_rows == 3 + assert aligned.selected_rows == 3 + assert aligned.candidates["id"].tolist() == ["b", "a", "c"] + assert aligned.candidates["grade"].tolist() == ["B", "A", "C"] + assert aligned.intervals["id"].tolist() == ["b", "a", "c"] + assert aligned.intervals["grade"].tolist() == ["score_q02", "score_q01", "score_q03"] + np.testing.assert_allclose(aligned.intervals["y_pred"], [0.2, 0.1, 0.3]) + + +def test_id_alignment_sampling_is_sorted_and_reproducible() -> None: + candidates = pd.DataFrame({"id": np.arange(20), "row": np.arange(20)}) + intervals = pd.DataFrame( + { + "id": np.arange(19, -1, -1), + "y_pred": np.arange(19, -1, -1) / 100.0, + } + ) + expected_positions = np.sort(np.random.default_rng(17).choice(20, size=6, replace=False)) + + first = align_candidate_intervals( + candidates, + intervals, + max_candidates=6, + random_state=17, + ) + second = align_candidate_intervals( + candidates, + intervals, + max_candidates=6, + random_state=17, + ) + + assert first.candidates["row"].tolist() == expected_positions.tolist() + pd.testing.assert_frame_equal(first.candidates, second.candidates) + pd.testing.assert_frame_equal(first.intervals, second.intervals) + + +def test_row_number_alignment_reorders_interval_rows_without_losing_source_columns() -> None: + candidates = pd.DataFrame({"loan": ["a", "b", "c"], "grade": ["A", "B", "C"]}) + intervals = pd.DataFrame( + { + "_row_number": [2, 0, 1], + "grade": ["q3", "q1", "q2"], + "y_pred": [0.3, 0.1, 0.2], + } + ) + + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=0, + random_state=42, + ) + + assert aligned.mode == "row_number" + assert aligned.candidates["loan"].tolist() == ["a", "b", "c"] + assert aligned.intervals["_row_number"].tolist() == [0, 1, 2] + assert aligned.intervals["grade"].tolist() == ["q1", "q2", "q3"] + + +def test_positional_fallback_samples_from_full_alignable_universe() -> None: + candidates = pd.DataFrame({"row": np.arange(10)}) + intervals = pd.DataFrame({"y_pred": np.arange(10) / 100.0}) + expected_positions = np.sort(np.random.default_rng(7).choice(10, size=4, replace=False)) + + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=4, + random_state=7, + ) + + assert aligned.mode == "position" + assert aligned.available_rows == 10 + assert aligned.candidates["row"].tolist() == expected_positions.tolist() + np.testing.assert_allclose( + aligned.intervals["y_pred"], + expected_positions / 100.0, + ) + assert max(expected_positions) >= 4 + + +@pytest.mark.parametrize( + ("candidate_ids", "interval_ids", "message"), + [ + (["a", "a"], ["a", "b"], "candidate id alignment key is not unique"), + (["a", "b"], ["a", "a"], "interval id alignment key is not unique"), + (["a", None], ["a", "b"], "candidate id alignment key contains missing"), + (["a", "b"], ["a", None], "interval id alignment key contains missing"), + ], +) +def test_id_alignment_rejects_ambiguous_keys( + candidate_ids: list[str | None], + interval_ids: list[str | None], + message: str, +) -> None: + candidates = pd.DataFrame({"id": candidate_ids}) + intervals = pd.DataFrame({"id": interval_ids, "y_pred": [0.1, 0.2]}) + + with pytest.raises(ValueError, match=message): + align_candidate_intervals( + candidates, + intervals, + max_candidates=None, + random_state=42, + ) + + +def test_id_alignment_rejects_disjoint_universes() -> None: + candidates = pd.DataFrame({"id": ["a", "b"]}) + intervals = pd.DataFrame({"id": ["c", "d"], "y_pred": [0.1, 0.2]}) + + with pytest.raises(ValueError, match="produced zero rows"): + align_candidate_intervals( + candidates, + intervals, + max_candidates=None, + random_state=42, + ) diff --git a/tests/test_optimization/test_policy.py b/tests/test_optimization/test_policy.py index 1212627..c7eb5cb 100644 --- a/tests/test_optimization/test_policy.py +++ b/tests/test_optimization/test_policy.py @@ -9,11 +9,18 @@ from __future__ import annotations import numpy as np +import pandas as pd import pytest from hypothesis import HealthCheck, given, settings, strategies as st from hypothesis.extra.numpy import arrays -from src.optimization.policy import PolicyMode, all_policy_modes, resolve_policy_mode +from src.optimization.policy import ( + PolicyMode, + all_policy_modes, + policy_segment_labels, + policy_uses_segment_labels, + resolve_policy_mode, +) from src.optimization.portfolio_model import compute_effective_pd # --------------------------------------------------------------------------- @@ -64,6 +71,31 @@ def test_resolve_policy_mode_defaults_for_none_or_empty() -> None: assert resolve_policy_mode("") is PolicyMode.HARD_WORST_CASE +def test_only_segment_policies_request_segment_labels() -> None: + assert policy_uses_segment_labels(PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY) + assert policy_uses_segment_labels(PolicyMode.SEGMENT_RELATIVE_TAIL_BLENDED_UNCERTAINTY) + assert not policy_uses_segment_labels(PolicyMode.BLENDED_UNCERTAINTY) + + +def test_policy_segment_labels_use_one_canonical_fallback_contract() -> None: + loans = pd.DataFrame( + { + "original_grade": ["A", None], + "term": [36, 60], + } + ) + + labels = policy_segment_labels( + loans, + PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY, + grade_column="original_grade", + ) + + assert labels is not None + assert labels.tolist() == ["A|36|unknown", "unknown|60|unknown"] + assert policy_segment_labels(loans, PolicyMode.BLENDED_UNCERTAINTY) is None + + # --------------------------------------------------------------------------- # compute_effective_pd — invariants per policy # --------------------------------------------------------------------------- @@ -159,6 +191,43 @@ def test_every_policy_produces_pd_in_unit_interval(mode: PolicyMode) -> None: assert np.all((out >= 0.0) & (out <= 1.0)) +def test_segment_tail_blended_uses_segment_cutoffs_when_segments_are_large() -> None: + point = np.full(6, 0.10) + high = point + np.array([0.01, 0.04, 0.07, 0.02, 0.06, 0.10]) + labels = np.array(["A", "A", "A", "B", "B", "B"]) + + out = compute_effective_pd( + point, + high, + policy_mode=PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY, + gamma=0.5, + tail_focus_quantile=0.5, + segment_labels=labels, + min_segment_size=3, + ) + + expected_delta = np.array([0.00, 0.04, 0.07, 0.00, 0.06, 0.10]) + assert np.allclose(out, point + 0.5 * expected_delta) + + +def test_segment_relative_tail_blended_ranks_by_relative_width() -> None: + point = np.array([0.10, 0.20, 0.10, 0.20]) + high = np.array([0.12, 0.26, 0.15, 0.22]) + labels = np.array(["A", "A", "B", "B"]) + + out = compute_effective_pd( + point, + high, + policy_mode=PolicyMode.SEGMENT_RELATIVE_TAIL_BLENDED_UNCERTAINTY, + gamma=1.0, + tail_focus_quantile=0.5, + segment_labels=labels, + min_segment_size=1, + ) + + assert np.allclose(out, np.array([0.10, 0.26, 0.15, 0.20])) + + def test_legacy_string_aliases_still_work() -> None: """Backward compatibility: passing a legacy string must equal passing the enum.""" rng = np.random.default_rng(seed=7) diff --git a/tests/test_optimization/test_policy_evaluation.py b/tests/test_optimization/test_policy_evaluation.py new file mode 100644 index 0000000..45a5866 --- /dev/null +++ b/tests/test_optimization/test_policy_evaluation.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd + +from src.optimization import policy_evaluation +from src.optimization.policy import PolicyMode + + +def _inputs() -> dict[str, Any]: + loans = pd.DataFrame( + { + "loan_amnt": [100.0, 200.0], + "grade": ["A", "B"], + "term": [36, 60], + "verification_status": ["Verified", "Not Verified"], + } + ) + return { + "loans": loans, + "pd_point": np.array([0.10, 0.20]), + "pd_low": np.array([0.05, 0.10]), + "pd_high": np.array([0.30, 0.50]), + "lgd": np.array([0.45, 0.45]), + "int_rates": np.array([0.12, 0.15]), + } + + +def test_solve_policy_allocation_resolves_effective_pd_once(monkeypatch) -> None: + captured: dict[str, Any] = {} + + def fake_optimize(**kwargs): + captured.update(kwargs) + return { + "allocation_vector": np.array([1.0, 0.5]), + "objective_value": 1.0, + "n_funded": 2, + "total_allocated": 200.0, + } + + monkeypatch.setattr(policy_evaluation, "optimize_portfolio_allocation", fake_optimize) + result = policy_evaluation.solve_policy_allocation( + **_inputs(), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.5, + ) + + assert result.policy_mode is PolicyMode.BLENDED_UNCERTAINTY + assert np.allclose(result.effective_pd, np.array([0.20, 0.35])) + assert np.allclose(captured["pd_constraint_override"], result.effective_pd) + assert np.allclose(captured["pd_point"], _inputs()["pd_point"]) + assert result.objective_risk_mode == "point_pd_plus_aversion" + assert np.allclose(result.allocation, np.array([1.0, 0.5])) + + +def test_nonrobust_policy_always_uses_point_pd(monkeypatch) -> None: + captured: dict[str, Any] = {} + + def fake_optimize(**kwargs): + captured.update(kwargs) + return { + "allocation_vector": np.array([0.0, 1.0]), + "objective_value": 1.0, + "n_funded": 1, + "total_allocated": 200.0, + } + + monkeypatch.setattr(policy_evaluation, "optimize_portfolio_allocation", fake_optimize) + result = policy_evaluation.solve_policy_allocation( + **_inputs(), + robust=False, + policy_mode=PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY, + gamma=0.9, + uncertainty_aversion=0.5, + ) + + assert result.policy_mode is PolicyMode.POINT_ESTIMATE + assert result.gamma == 0.0 + assert np.allclose(result.effective_pd, _inputs()["pd_point"]) + assert np.allclose(captured["pd_constraint_override"], _inputs()["pd_point"]) + assert captured["uncertainty_aversion"] == 0.0 + + +def test_uncertainty_aversion_keeps_point_pd_objective(monkeypatch) -> None: + captured: dict[str, Any] = {} + + def fake_optimize(**kwargs): + captured.update(kwargs) + return { + "allocation_vector": np.array([1.0, 0.0]), + "objective_value": 1.0, + "n_funded": 1, + "total_allocated": 100.0, + } + + monkeypatch.setattr(policy_evaluation, "optimize_portfolio_allocation", fake_optimize) + result = policy_evaluation.solve_policy_allocation( + **_inputs(), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.5, + uncertainty_aversion=0.05, + ) + + assert np.allclose(captured["pd_point"], _inputs()["pd_point"]) + assert np.allclose(captured["pd_constraint_override"], result.effective_pd) + assert captured["uncertainty_aversion"] == 0.05 + assert result.objective_risk_mode == "point_pd_plus_aversion" diff --git a/tests/test_optimization/test_policy_selection.py b/tests/test_optimization/test_policy_selection.py new file mode 100644 index 0000000..d485ff6 --- /dev/null +++ b/tests/test_optimization/test_policy_selection.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from src.optimization.policy_selection import ( + build_linear_policy_grid, + endpoint_cap_stability, + select_policy_result_ex_ante, + temporal_period_labels, +) + + +def _selection_results() -> pd.DataFrame: + return pd.DataFrame( + { + "candidate_id": ["linear-001", "linear-002", "linear-003"], + "solver_status": ["Optimal", "Optimal", "Optimal"], + "expected_objective": [120.0, 110.0, 100.0], + "endpoint_budget": [0.40, 0.27, 0.20], + "weighted_pd_effective": [0.17, 0.17, 0.17], + "risk_tolerance": [0.17, 0.17, 0.17], + "total_allocated": [1_000.0, 1_000.0, 1_000.0], + } + ) + + +def test_round_grid_is_deterministic() -> None: + grid = build_linear_policy_grid( + risk_tolerances=[0.15, 0.17, 0.19], + gammas=[0.25, 0.50, 0.75], + uncertainty_aversions=[0.0], + ) + + assert len(grid) == 9 + assert grid[4].candidate_id == "linear-005" + assert grid[4].risk_tolerance == 0.17 + assert grid[4].gamma == 0.50 + + +def test_selector_uses_expected_objective_inside_screen() -> None: + selected, audit = select_policy_result_ex_ante( + _selection_results(), + endpoint_budget_cap=0.28, + budget=1_000.0, + ) + + assert selected["candidate_id"] == "linear-002" + assert audit["n_eligible"] == 2 + assert audit["outcome_columns_used"] == 0 + assert audit["statistical_assumption_columns_used"] == 0 + + +def test_selector_does_not_accept_suboptimal_status() -> None: + results = _selection_results() + results.loc[0, "solver_status"] = "Suboptimal" + results.loc[0, "endpoint_budget"] = 0.20 + + selected, _ = select_policy_result_ex_ante( + results, + endpoint_budget_cap=0.28, + budget=1_000.0, + ) + + assert selected["candidate_id"] == "linear-002" + + +def test_selector_rejects_duplicate_candidates() -> None: + results = pd.concat([_selection_results(), _selection_results().iloc[[0]]]) + + with pytest.raises(ValueError, match="duplicate candidate_id"): + select_policy_result_ex_ante( + results, + endpoint_budget_cap=0.28, + budget=1_000.0, + ) + + +def test_selector_rejects_outcome_derived_columns() -> None: + results = _selection_results().assign(realized_return=[1.0, 2.0, 3.0]) + + with pytest.raises(ValueError, match="outcome-derived"): + select_policy_result_ex_ante( + results, + endpoint_budget_cap=0.28, + budget=1_000.0, + ) + + +def test_selector_rejects_invalid_budget_utilization() -> None: + with pytest.raises(ValueError, match="min_budget_utilization"): + select_policy_result_ex_ante( + _selection_results(), + endpoint_budget_cap=0.28, + budget=1_000.0, + min_budget_utilization=1.01, + ) + + +def test_endpoint_cap_stability_finds_next_higher_objective_boundary() -> None: + results = _selection_results() + selected, _ = select_policy_result_ex_ante( + results, + endpoint_budget_cap=0.28, + budget=1_000.0, + ) + + stability = endpoint_cap_stability( + results, + selected_candidate_id=str(selected["candidate_id"]), + endpoint_budget_cap=0.28, + budget=1_000.0, + ) + + assert stability["cap_lower_inclusive"] == pytest.approx(0.27) + assert stability["cap_upper_exclusive"] == pytest.approx(0.40) + assert stability["margin_to_lower_boundary"] == pytest.approx(0.01) + assert stability["margin_to_upper_boundary"] == pytest.approx(0.12) + + +def test_temporal_labels_pool_late_years() -> None: + dates = pd.Series(["2018-01-01", "2018-08-01", "2020-03-01", "2021-09-01"]) + + assert temporal_period_labels(dates).tolist() == [ + "2018H1", + "2018H2", + "2020+", + "2020+", + ] diff --git a/tests/test_optimization/test_portfolio_model.py b/tests/test_optimization/test_portfolio_model.py index e609f9d..53ed6ce 100644 --- a/tests/test_optimization/test_portfolio_model.py +++ b/tests/test_optimization/test_portfolio_model.py @@ -5,7 +5,10 @@ import pytest import src.optimization.portfolio_model as portfolio_model -from src.optimization.portfolio_model import optimize_portfolio_allocation +from src.optimization.portfolio_model import ( + optimize_portfolio_allocation, + solution_allocation_vector, +) def _toy_loans() -> tuple[pd.DataFrame, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: @@ -28,6 +31,36 @@ def _weighted(values: np.ndarray, allocation: np.ndarray, amounts: np.ndarray) - return float(np.sum(allocation * amounts * values) / exposure) +def test_solution_allocation_vector_normalizes_dense_and_sparse_payloads() -> None: + dense = solution_allocation_vector( + {"allocation_vector": np.array([0.25, 0.0, 1.0])}, + 3, + ) + sparse = solution_allocation_vector({"allocation": {0: 0.25, 2: 1.0}}, 3) + + np.testing.assert_allclose(dense, [0.25, 0.0, 1.0]) + np.testing.assert_allclose(sparse, dense) + + +@pytest.mark.parametrize( + ("solution", "n_items", "error", "match"), + [ + ({"allocation_vector": [0.5, 0.5]}, 3, ValueError, "shape mismatch"), + ({"allocation": {0: float("nan")}}, 1, ValueError, "non-finite"), + ({}, 1, TypeError, "allocation_vector"), + ({"allocation": {}}, -1, ValueError, "nonnegative"), + ], +) +def test_solution_allocation_vector_rejects_invalid_payloads( + solution: dict[str, object], + n_items: int, + error: type[Exception], + match: str, +) -> None: + with pytest.raises(error, match=match): + solution_allocation_vector(solution, n_items) + + def test_highs_sparse_respects_portfolio_constraints() -> None: loans, pd_point, pd_low, pd_high, lgd, int_rates = _toy_loans() result = optimize_portfolio_allocation( @@ -80,6 +113,32 @@ def test_highs_sparse_matches_pyomo_highs_objective_on_toy_lp() -> None: assert sparse["total_allocated"] == pytest.approx(pyomo["total_allocated"], rel=1e-6) +def test_highs_sparse_matches_pyomo_when_pd_slack_is_enabled() -> None: + loans, pd_point, pd_low, pd_high, lgd, int_rates = _toy_loans() + kwargs = { + "loans": loans, + "pd_point": pd_point, + "pd_low": pd_low, + "pd_high": pd_high, + "lgd": lgd, + "int_rates": int_rates, + "total_budget": 3500, + "max_concentration": 0.75, + "max_portfolio_pd": 0.03, + "robust": True, + "min_budget_utilization": 0.80, + "pd_cap_slack_penalty": 0.01, + "pd_constraint_override": pd_high, + } + + sparse = optimize_portfolio_allocation(**kwargs, solver_backend="highs_sparse") + pyomo = optimize_portfolio_allocation(**kwargs, solver_backend="highs_pyomo") + + assert sparse["objective_value"] == pytest.approx(pyomo["objective_value"], rel=1e-6) + assert sparse["total_allocated"] == pytest.approx(pyomo["total_allocated"], rel=1e-6) + assert float(sparse["pd_cap_slack"]) == pytest.approx(float(pyomo["pd_cap_slack"]), rel=1e-6) + + def test_highspy_matches_sparse_highs_objective_on_toy_lp() -> None: pytest.importorskip("highspy") loans, pd_point, pd_low, pd_high, lgd, int_rates = _toy_loans() diff --git a/tests/test_pool93_body_claim_sync.py b/tests/test_pool93_body_claim_sync.py index dcb8d56..50054de 100644 --- a/tests/test_pool93_body_claim_sync.py +++ b/tests/test_pool93_body_claim_sync.py @@ -1,21 +1,7 @@ -"""Drift guard for the promoted pool93 IJDS body claim. - -The paper body point (A35 "Body/default balanced point") lives in the pool93 -governance sidecars and the A35-A39 tables, all generated outside the DVC DAG -by the champion-reopen experiment scripts. The IJDS manuscript embeds those -numbers as hand-written prose/Markdown, so a regenerated CSV or a retyped -figure can silently desync the submission from its evidence. These tests lock -the two-tag scheme together: the frozen rebaseline chain stays the declared -return floor, and the pool93 governance JSONs stay the authoritative source -for every body-claim number printed in the paper surfaces. - -Only anchor values are checked -- enough to catch a stale copy/paste, without -re-implementing a Markdown table parser. -""" +"""Historical integrity checks for the manifest-protected pool93 bundle.""" from __future__ import annotations -import csv import json from pathlib import Path from typing import Any @@ -24,218 +10,57 @@ REPO = Path(__file__).resolve().parents[1] TABLES = REPO / "reports" / "crpto" / "tables" - -PAPER = REPO / "paper" / "CRPTO_ijds.qmd" -SUPPLEMENT = REPO / "paper" / "supplement_ijds.qmd" - +MANIFEST = REPO / "EXTRACTION_MANIFEST.json" +PROMOTION = REPO / "models" / "final_project_promotion.json" TERMINAL_GOVERNANCE = ( REPO - / "models" - / "experiments" - / "champion_reopen" + / "models/experiments/champion_reopen" / "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" - / "portfolio" - / "pool93_ijds_claim_governance.json" + / "portfolio/pool93_ijds_claim_governance.json" ) CONSOLIDATED_GOVERNANCE = ( REPO - / "models" - / "experiments" - / "champion_reopen" - / "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" - / "portfolio" - / "pool93_ijds_consolidated_governance.json" + / "models/experiments/champion_reopen" + / "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" + / "portfolio/pool93_ijds_consolidated_governance.json" ) -PROMOTION = REPO / "models" / "final_project_promotion.json" -MANIFEST = REPO / "EXTRACTION_MANIFEST.json" - POOL93_TABLE_STEMS = ( "crpto_tableA35_pool93_ijds_frontier", "crpto_tableA36_pool93_body_funded_grade_audit", "crpto_tableA37_pool93_body_tail_risk", "crpto_tableA38_pool93_body_cluster_bound_audit", "crpto_tableA39_pool93_body_bootstrap_metrics", + "crpto_tableA40_pool93_point_baseline", ) -TERMINAL_RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" -REBASELINE_RUN_TAG = "ijds-rebaseline-2026-06-07" - -EXPECTED_BODY = { - "return": 184832.475845, - "Gamma_CP": 0.162616, - "V": 0.03535, - "Markov_cap": 0.34508374, - "endpoint_budget_upper": 0.24508374, - "risk_tolerance": 0.1715, - "gamma": 0.5475, - "uncertainty_aversion": 0.05, - "alpha_pass": "8/8", -} -EXPECTED_FLOOR = 170464.54 -EXPECTED_FROZEN_RETURN = 170464.5429284627 -EXPECTED_FRONTIER_COUNTS = { - "raw_rows": 51678, - "deduped_semantic_policies": 50010, - "eligible_all_alpha_return_floor_policies": 27508, -} -EXPECTED_TERMINAL_COUNTS = { - "n_policies": 37068, - "n_all_alpha_passers": 37068, - "n_all_alpha_passers_above_return_floor": 14814, -} - -def _load_json(path: Path) -> dict[str, Any]: +def _load(path: Path) -> dict[str, Any]: if not path.is_file(): - pytest.skip(f"{path.name} not present locally.") + pytest.skip(f"Historical artifact unavailable: {path}") return json.loads(path.read_text(encoding="utf-8")) -def _read_rows(name: str) -> list[dict[str, str]]: - path = TABLES / name - if not path.is_file(): - pytest.skip(f"{name} not present locally.") - with path.open(encoding="utf-8") as fh: - return list(csv.DictReader(fh)) - - -def _text(path: Path) -> str: - if not path.is_file(): - pytest.skip(f"{path} not present locally.") - return path.read_text(encoding="utf-8") +def test_historical_pool93_bundle_remains_internally_coherent() -> None: + consolidated = _load(CONSOLIDATED_GOVERNANCE) + terminal = _load(TERMINAL_GOVERNANCE) + promotion = _load(PROMOTION) - -def _body_row_a35() -> dict[str, str]: - rows = _read_rows("crpto_tableA35_pool93_ijds_frontier.csv") - matches = [r for r in rows if r["role"] == "Body/default balanced point"] - assert len(matches) == 1, "A35 must expose exactly one body/default balanced row" - return matches[0] - - -def test_pool93_claim_governance_matches_expected_body_point() -> None: - """Consolidated governance body point and A35 body row agree with the claim.""" - consolidated = _load_json(CONSOLIDATED_GOVERNANCE) - body = consolidated["selected_candidates"]["paper_body"] - assert body["return"] == pytest.approx(EXPECTED_BODY["return"], abs=1e-6) - assert body["Gamma_CP"] == pytest.approx(EXPECTED_BODY["Gamma_CP"], abs=1e-9) - assert body["V"] == pytest.approx(EXPECTED_BODY["V"], abs=1e-9) - assert body["Markov_cap"] == pytest.approx(EXPECTED_BODY["Markov_cap"], abs=1e-9) - assert body["endpoint_budget_upper"] == pytest.approx( - EXPECTED_BODY["endpoint_budget_upper"], abs=1e-9 + assert consolidated["counts"]["deduped_semantic_policies"] == 50_010 + assert consolidated["counts"]["eligible_all_alpha_return_floor_policies"] == 27_508 + assert terminal["runtime_status"]["total_checks"] == 296_544 + assert promotion["run_tag"] == "ijds-rebaseline-2026-06-07" + assert terminal["claim_summary"]["declared_return_floor"] == pytest.approx( + round(promotion["final_champion"]["realized_total_return"], 2) ) - assert body["risk_tolerance"] == EXPECTED_BODY["risk_tolerance"] - assert body["gamma"] == EXPECTED_BODY["gamma"] - assert body["uncertainty_aversion"] == EXPECTED_BODY["uncertainty_aversion"] - assert body["alpha_pass"] == EXPECTED_BODY["alpha_pass"] - - terminal = _load_json(TERMINAL_GOVERNANCE) - assert terminal["run_tag"] == TERMINAL_RUN_TAG - summary = terminal["claim_summary"] - assert summary["declared_return_floor"] == EXPECTED_FLOOR - assert summary["finite_grid_policy"]["alpha_grid_size"] == 8 - row = _body_row_a35() - assert float(row["realized_return"]) == pytest.approx(body["return"], abs=1e-6) - assert float(row["Gamma_CP_alpha01"]) == pytest.approx(body["Gamma_CP"], abs=1e-9) - assert float(row["V_alpha01"]) == pytest.approx(body["V"], abs=1e-9) - assert float(row["Markov_cap_alpha01"]) == pytest.approx(body["Markov_cap"], abs=1e-9) - assert row["alpha_grid_pass"] == body["alpha_pass"] - - -def test_pool93_consolidated_governance_frontier_counts() -> None: - """Frontier and terminal-search counts quoted in the paper match governance.""" - consolidated = _load_json(CONSOLIDATED_GOVERNANCE) - for key, expected in EXPECTED_FRONTIER_COUNTS.items(): - assert consolidated["counts"][key] == expected, key - - terminal = _load_json(TERMINAL_GOVERNANCE) - counts = terminal["claim_hierarchy"]["current_counts"] - for key, expected in EXPECTED_TERMINAL_COUNTS.items(): - assert counts[key] == expected, key - assert terminal["runtime_status"]["total_checks"] == 296544 - - -def test_pool93_tables_exist() -> None: - """A35-A39 evidence tables exist in both CSV and TEX form.""" - missing = [ - f"{stem}.{ext}" - for stem in POOL93_TABLE_STEMS - for ext in ("csv", "tex") - if not (TABLES / f"{stem}.{ext}").is_file() - ] - assert not missing, "pool93 evidence tables missing: " + ", ".join(missing) - - -def test_pool93_paper_anchors_match_csvs() -> None: - """Body-claim numbers printed in the paper surfaces derive from A35/A39.""" - row = _body_row_a35() - budget = float(row["endpoint_budget_upper_alpha01"]) - deterministic_bound = budget + float(row["V_alpha01"]) - paper_anchors = [ - f"${float(row['realized_return']):,.2f}", - f"{float(row['V_alpha01']):.6f}", - f"{float(row['Gamma_CP_alpha01']):.6f}", - f"{float(row['Markov_cap_alpha01']):.6f}", - f"{budget:.6f}", - f"{deterministic_bound:.6f}", - ] - - boot = {r["metric"]: r for r in _read_rows("crpto_tableA39_pool93_body_bootstrap_metrics.csv")} - return_boot = boot["funded_set_repriced_return_lgd45"] - supplement_anchors = [ - *paper_anchors, - f"${float(return_boot['boot_p025']):,.2f}", - f"${float(return_boot['boot_p975']):,.2f}", - ] - - missing: list[str] = [] - paper_text = _text(PAPER) - missing.extend(f"{a} missing in {PAPER.name}" for a in paper_anchors if a not in paper_text) - supplement_text = _text(SUPPLEMENT) - missing.extend( - f"{a} missing in {SUPPLEMENT.name}" for a in supplement_anchors if a not in supplement_text - ) - assert not missing, "pool93 body-claim drift:\n" + "\n".join(missing) - - -def test_pool93_two_tag_scheme_is_coherent() -> None: - """The frozen rebaseline chain stays the declared return floor for pool93.""" - promotion = _load_json(PROMOTION) - assert promotion["run_tag"] == REBASELINE_RUN_TAG - frozen_return = promotion["final_champion"]["realized_total_return"] - assert frozen_return == pytest.approx(EXPECTED_FROZEN_RETURN, abs=1e-6) - - terminal = _load_json(TERMINAL_GOVERNANCE) - floor = terminal["claim_summary"]["declared_return_floor"] - assert floor == pytest.approx(round(frozen_return, 2), abs=1e-9) - - -def test_pool93_manifest_block_agrees() -> None: - """EXTRACTION_MANIFEST pool93 block mirrors the governance sidecars.""" - manifest = _load_json(MANIFEST) - block = manifest.get("pool93_ijds_promotion") - if block is None: - pytest.skip("pool93_ijds_promotion block not yet added to EXTRACTION_MANIFEST.json.") - - consolidated = _load_json(CONSOLIDATED_GOVERNANCE) - body = consolidated["selected_candidates"]["paper_body"] - point = block["paper_body_point"] - assert block["terminal_run_tag"] == TERMINAL_RUN_TAG - assert point["realized_total_return"] == pytest.approx(body["return"], abs=1e-6) - assert point["alpha01_gamma_cp"] == pytest.approx(body["Gamma_CP"], abs=1e-9) - assert point["alpha01_weighted_miscoverage_V"] == pytest.approx(body["V"], abs=1e-9) - assert point["markov_cap_alpha01"] == pytest.approx(body["Markov_cap"], abs=1e-9) - assert point["declared_return_floor"] == EXPECTED_FLOOR - assert ( - block["frontier_counts"]["deduped_semantic_policies"] - == (EXPECTED_FRONTIER_COUNTS["deduped_semantic_policies"]) - ) +def test_historical_pool93_tables_remain_hash_protected() -> None: + manifest = _load(MANIFEST) hashed = set(manifest["critical_hashes"]) - missing = [ - f"reports/crpto/tables/{stem}.{ext}" + expected = { + f"reports/crpto/tables/{stem}.{suffix}" for stem in POOL93_TABLE_STEMS - for ext in ("csv", "tex") - if f"reports/crpto/tables/{stem}.{ext}" not in hashed - ] - assert not missing, "pool93 tables not hash-protected: " + ", ".join(missing) + for suffix in ("csv", "tex") + } + assert expected.issubset(hashed) + assert all((TABLES / path.split("/")[-1]).is_file() for path in expected) diff --git a/tests/test_publication_integrity.py b/tests/test_publication_integrity.py new file mode 100644 index 0000000..c736c1f --- /dev/null +++ b/tests/test_publication_integrity.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from scripts.check_publication_integrity import check_publication_integrity + + +def test_active_ijds_publication_surfaces_are_claim_synchronized() -> None: + assert check_publication_integrity() == [] diff --git a/tests/test_publication_targets.py b/tests/test_publication_targets.py index 4f29ac4..2cb08e8 100644 --- a/tests/test_publication_targets.py +++ b/tests/test_publication_targets.py @@ -46,11 +46,13 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No backlog = pack["backlog_not_blocking"] assert "no longer a blanket exclusion" in boundary - assert "future work" in boundary + assert "outside the submitted claim" in boundary + assert "not acceptance criteria" in boundary assert set(included) == { "regret_auditability_frontier", "tail_risk_oce_cvar_diagnostic", - "pool93_frontier_and_selected_allocation", + "exact_alpha_calibration_selected_policy", + "matched_point_pd_baseline", "robust_satisficing_margins", "dependence_aware_bound", "tail_satisficing_challenger_audit", @@ -60,9 +62,10 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No } assert included["regret_auditability_frontier"]["status"] == "include_body" assert included["tail_risk_oce_cvar_diagnostic"]["status"] == "include_supplement" - assert included["pool93_frontier_and_selected_allocation"]["status"] == ( + assert included["exact_alpha_calibration_selected_policy"]["status"] == ( "include_body_and_supplement" ) + assert included["matched_point_pd_baseline"]["status"] == ("include_body_and_supplement") assert included["robust_satisficing_margins"]["status"] == ("include_supplement_or_short_body") assert included["dependence_aware_bound"]["status"] == "include_theory_appendix_or_caveat" assert included["tail_satisficing_challenger_audit"]["status"] == "include_supplement" @@ -71,12 +74,15 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No assert included["multidataset_external_replication"]["status"] == ( "include_supplement_or_short_body" ) - pool93_artifacts = included["pool93_frontier_and_selected_allocation"]["artifacts"] - assert "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv" in pool93_artifacts - assert "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv" in ( - pool93_artifacts + active_artifacts = included["exact_alpha_calibration_selected_policy"]["artifacts"] + assert "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv" in active_artifacts + assert "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv" in ( + active_artifacts ) - for artifact in pool93_artifacts: + assert "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv" in ( + active_artifacts + ) + for artifact in active_artifacts: assert Path(artifact).exists(), artifact multidataset_artifacts = included["multidataset_external_replication"]["artifacts"] assert "reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.csv" in ( @@ -99,7 +105,9 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No paper_readme = Path("paper/README.md").read_text(encoding="utf-8") for text in (body, supplement, paper_readme): - assert "regret-auditability" in text.lower() - assert "OCE/CVaR" in text - assert "satisficing" in text.lower() - assert "multi-dataset" in text.lower() + assert "midpoint" in text.lower() + assert "calibration" in text.lower() + assert "point-PD" in text + + for diagnostic in ("OCE/CVaR", "SPO+", "Prosper"): + assert diagnostic in supplement diff --git a/tests/test_scripts/test_benchmark_conformal_variants.py b/tests/test_scripts/test_benchmark_conformal_variants.py index d662c25..b3f55e2 100644 --- a/tests/test_scripts/test_benchmark_conformal_variants.py +++ b/tests/test_scripts/test_benchmark_conformal_variants.py @@ -2,7 +2,7 @@ from __future__ import annotations -from scripts.benchmark_conformal_variants import _build_output_paths +from scripts.benchmark_conformal_variants import _build_output_paths, _normalize_search_space def test_build_output_paths_uses_namespaced_shadow_locations() -> None: @@ -15,3 +15,24 @@ def test_build_output_paths_uses_namespaced_shadow_locations() -> None: .as_posix() .endswith("models/conformal_gap/abc_def/conformal_variant_selection_status.json") ) + + +def test_normalize_search_space_dedupes_and_applies_defaults() -> None: + space = _normalize_search_space( + calibration_size_fractions=(0.25, 1.5, 0.5), + partition_candidates=(" grade ", "grade", ""), + partition_probability_sources=("RAW", "raw"), + n_score_bins_candidates=(0, 10), + fallback_modes=("", "Grade_Then_Global", "grade_then_global"), + score_scale_families=("",), + min_group_sizes=None, + min_group_size_default=500, + ) + + assert space.partition_candidates == ("grade",) + assert space.partition_probability_sources == ("raw",) + assert space.n_score_bins_candidates == (10,) + assert space.fallback_modes == ("grade_then_global",) + assert space.score_scale_families == ("none",) + assert space.min_group_sizes == (500,) + assert space.calibration_size_fractions == (0.25, 0.5) diff --git a/tests/test_scripts/test_benchmark_pd_set_prediction.py b/tests/test_scripts/test_benchmark_pd_set_prediction.py new file mode 100644 index 0000000..2188404 --- /dev/null +++ b/tests/test_scripts/test_benchmark_pd_set_prediction.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import numpy as np + +from scripts.benchmark_pd_set_prediction import ( + _build_output_paths, + _promotion_gate, + _set_benchmark_settings, +) + + +def test_build_output_paths_uses_namespaced_shadow_locations() -> None: + paths = _build_output_paths("set/audit") + + assert ( + paths["cases"] + .as_posix() + .endswith("data/processed/conformal_gap/set_audit/pd_set_prediction_cases.parquet") + ) + assert ( + paths["status"] + .as_posix() + .endswith("models/conformal_gap/set_audit/pd_set_prediction_status.json") + ) + + +def test_set_benchmark_settings_normalizes_inputs_and_fallback() -> None: + settings = _set_benchmark_settings( + alpha=0.1, + method="lac", + methods=("lac", "aps", "lac"), + partitions=("global", "global", "grade"), + partition_probability_source="RAW", + n_score_bins=10, + min_group_size=500, + fallback_mode="score_only", + calibration_size_fractions=(0.25, 1.5, 0.50), + prob_cal_lookup={"raw": np.array([0.1]), "calibrated": np.array([0.2])}, + ) + + assert settings.methods == ("lac", "aps") + assert settings.partitions == ("global", "grade") + assert settings.partition_probability_source == "raw" + assert settings.effective_fallback_mode == "global_only" + assert settings.calibration_size_fractions == (0.25, 0.50) + + +def test_promotion_gate_requires_coverage_grade_a_and_breadth() -> None: + gate = _promotion_gate( + {"set_coverage": 0.90}, + [ + {"slice_value": "A", "singleton_rate": 0.81}, + {"slice_value": "B", "singleton_rate": 0.41}, + {"slice_value": "C", "singleton_rate": 0.42}, + {"slice_value": "D", "singleton_rate": 0.43}, + ], + ) + + assert gate["pass"] is True + assert gate["grade_a_singleton_rate"] == 0.81 + assert gate["grades_with_singleton_above_40pct"] == 4 diff --git a/tests/test_scripts/test_build_crpto_journal_package.py b/tests/test_scripts/test_build_crpto_journal_package.py index 7cc4bab..88a919d 100644 --- a/tests/test_scripts/test_build_crpto_journal_package.py +++ b/tests/test_scripts/test_build_crpto_journal_package.py @@ -2,6 +2,7 @@ import subprocess import sys +import time from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path @@ -9,6 +10,25 @@ import pytest +def _restore_payload(path: Path, payload: bytes | None) -> None: + for attempt in range(5): + try: + if payload is None: + path.unlink(missing_ok=True) + return + + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.restore.{attempt}.tmp") + tmp.write_bytes(payload) + tmp.replace(path) + return + except OSError: + tmp.unlink(missing_ok=True) if "tmp" in locals() else None + if attempt == 4: + raise + time.sleep(0.2 * (attempt + 1)) + + @contextmanager def _preserve_files(paths: list[Path]) -> Iterator[None]: snapshots = {path: path.read_bytes() if path.exists() else None for path in paths} @@ -16,11 +36,7 @@ def _preserve_files(paths: list[Path]) -> Iterator[None]: yield finally: for path, payload in snapshots.items(): - if payload is None: - path.unlink(missing_ok=True) - else: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(payload) + _restore_payload(path, payload) def test_build_crpto_journal_package_script_runs() -> None: diff --git a/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py b/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py new file mode 100644 index 0000000..9443fa4 --- /dev/null +++ b/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from scripts.build_ijds_calibration_selected_evidence import ( + build_baseline_table, + build_bootstrap_table, + build_grade_table, +) + + +def _allocations() -> pd.DataFrame: + return pd.DataFrame( + { + "role": ["calibration_selected", "calibration_selected", "point_pd_matched_tau"], + "issue_d": ["2020-01-01", "2020-02-01", "2020-01-01"], + "grade": ["A", "B", "A"], + "funded_exposure": [100.0, 100.0, 200.0], + "funded_weight": [0.5, 0.5, 1.0], + "outcome": [0.0, 1.0, 0.0], + "miscoverage": [0.0, 1.0, 0.0], + "pd_point": [0.1, 0.2, 0.2], + "pd_effective": [0.2, 0.3, 0.2], + "pd_high": [0.4, 0.6, 0.8], + "int_rate": [0.1, 0.2, 0.1], + "realized_return_contribution": [10.0, -45.0, 20.0], + } + ) + + +def _evaluation() -> pd.DataFrame: + return pd.DataFrame( + [ + { + "period": "full_oot", + "role": "calibration_selected", + "n_panel": 10, + "n_funded": 2, + "total_allocated": 200.0, + "expected_objective": 5.0, + "realized_return": -35.0, + "weighted_outcome": 0.5, + "weighted_miscoverage": 0.5, + "weighted_pd_point": 0.15, + "weighted_pd_effective": 0.25, + "gamma_cp": 0.35, + "gamma_internalized": 0.10, + "gamma_residual": 0.25, + "endpoint_budget": 0.50, + "markov_loss_threshold": 0.80, + }, + { + "period": "full_oot", + "role": "point_pd_matched_tau", + "n_panel": 10, + "n_funded": 1, + "total_allocated": 200.0, + "expected_objective": 10.0, + "realized_return": 20.0, + "weighted_outcome": 0.0, + "weighted_miscoverage": 0.0, + "weighted_pd_point": 0.20, + "weighted_pd_effective": 0.20, + "gamma_cp": 0.60, + "gamma_internalized": 0.0, + "gamma_residual": 0.60, + "endpoint_budget": 0.80, + "markov_loss_threshold": 1.10, + }, + ] + ) + + +def test_grade_table_preserves_selected_exposure() -> None: + table = build_grade_table(_allocations()) + + assert table["exposure"].sum() == 200.0 + assert table["exposure_share"].sum() == 1.0 + + +def test_bootstrap_is_deterministic_and_uses_official_observed_values() -> None: + first = build_bootstrap_table(_allocations(), _evaluation(), n_draws=100, seed=7) + second = build_bootstrap_table(_allocations(), _evaluation(), n_draws=100, seed=7) + + pd.testing.assert_frame_equal(first, second) + assert set(first["bootstrap_unit"]) == {"origination_month", "funded_loan"} + observed = first.loc[first["bootstrap_unit"].eq("origination_month")].set_index("metric")[ + "observed" + ] + assert observed["realized_return"] == -35.0 + assert observed["Gamma_CP"] == 0.35 + + +def test_bootstrap_rejects_allocation_evaluation_drift() -> None: + evaluation = _evaluation() + evaluation.loc[evaluation["role"].eq("calibration_selected"), "realized_return"] = 1.0 + + with pytest.raises(ValueError, match="realized_return"): + build_bootstrap_table(_allocations(), evaluation, n_draws=10, seed=7) + + +def test_baseline_table_uses_selected_policy_as_zero_delta() -> None: + table = build_baseline_table(_evaluation()) + selected = table.loc[table["policy"].eq("Calibration-selected 50/50 CRPTO")].iloc[0] + + assert np.isclose(selected["return_delta_vs_selected"], 0.0) + assert np.isclose(selected["default_delta_vs_selected"], 0.0) diff --git a/tests/test_scripts/test_build_papers_tesis_deep_audit.py b/tests/test_scripts/test_build_papers_tesis_deep_audit.py new file mode 100644 index 0000000..ace49b5 --- /dev/null +++ b/tests/test_scripts/test_build_papers_tesis_deep_audit.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from pathlib import Path + +from scripts.build_papers_tesis_deep_audit import write_audit + + +def _row( + *, + relative_path: str, + decision: str, + action_required: str = "none_now", + bib_status: str = "existing", +) -> dict[str, object]: + return { + "folder": relative_path.split("/", 1)[0], + "relative_path": relative_path, + "title": f"Title {relative_path}", + "status": "published", + "primary_domain": "conformal risk control", + "bib_key": "paperkey", + "bib_status": bib_status, + "core_concepts": "coverage; decision risk", + "key_claims": "claim summary", + "conclusions": "use with finite-grid boundary", + "figures_tables_useful": "inspiration only", + "limitations": "not CRPTO evidence", + "decision": decision, + "action_required": action_required, + "crpto_value": "supports CRPTO framing", + "extended_lab_value": "future-work value", + "evidence_gate": "do not reopen champion", + "artifact_sink": "docs/research/example.md", + "stop_rule": "keep as literature unless claim changes", + "implementation_or_experiment": "none", + } + + +def test_write_audit_builds_editorial_sections(tmp_path: Path) -> None: + rows = [ + _row(relative_path="paper/promote.pdf", decision="promote_crpto_body"), + _row(relative_path="paper/append.pdf", decision="append_crpto_related_work"), + _row( + relative_path="supplement/experiment.pdf", + decision="append_tail_risk", + action_required="experiment_completed_appendix_diagnostic", + ), + _row( + relative_path="tesis/future.pdf", + decision="park_future_work", + bib_status="needs_bib_if_cited", + ), + ] + curated_visual_rows = [ + { + "relative_path": "paper/promote.pdf", + "caption_type": "figure", + "caption_index": 1, + "editorial_sink": "own schematic", + "why_useful": "layout inspiration", + "claim_boundary": "do not reproduce", + } + ] + audit_path = tmp_path / "audit.md" + + write_audit( + audit_path, + rows, + tmp_path / "matrix.csv", + tmp_path / "captions.csv", + tmp_path / "visuals.csv", + curated_visual_rows, + ) + + text = audit_path.read_text(encoding="utf-8") + + assert "# Papers_tesis Deep Audit" in text + assert "## Lectura integrada para Paper CRPTO" in text + assert "paper/promote.pdf" in text + assert "paper/append.pdf" in text + assert "supplement/experiment.pdf" in text + assert "tesis/future.pdf" in text + assert "| action_required | n |" in text + assert "experiment_completed_appendix_diagnostic" in text + assert "needs_bib_if_cited" in text + assert "no cambia el champion CRPTO" in text diff --git a/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py b/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py new file mode 100644 index 0000000..1e870c7 --- /dev/null +++ b/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest + +from scripts.search import build_pool93_ijds_consolidated_frontier as frontier + + +def test_load_leaderboards_rehydrates_policy_aware_tail_bound( + tmp_path: Path, + monkeypatch, +) -> None: + leaderboard_path = tmp_path / "leaderboard.parquet" + bound_path = tmp_path / "bound.parquet" + pd.DataFrame( + { + "local_candidate_id": [7], + "semantic_policy_key": ["tail-policy"], + "alpha01_endpoint_budget_upper": [0.50], + "alpha01_markov_loss_cap": [0.60], + } + ).to_parquet(leaderboard_path, index=False) + pd.DataFrame( + { + "local_candidate_id": [7], + "semantic_policy_key": ["tail-policy"], + "alpha": [0.01], + "risk_tolerance": [0.20], + "gamma_cp": [0.60], + "weighted_pd_point": [0.10], + "weighted_pd_constraint_used": [0.20], + "weighted_pd_high": [0.60], + "pd_cap_slack": [0.0], + } + ).to_parquet(bound_path, index=False) + monkeypatch.setattr(frontier, "_leaderboard_path", lambda _tag: leaderboard_path) + monkeypatch.setattr(frontier, "_bound_eval_path", lambda _tag: bound_path) + + row = frontier._load_leaderboards(["unit-tag"]).iloc[0] + + assert row["alpha01_gamma_internalized"] == pytest.approx(0.10) + assert row["alpha01_gamma_residual"] == pytest.approx(0.40) + assert row["alpha01_endpoint_budget"] == pytest.approx(0.60) + assert row["alpha01_endpoint_budget_upper"] == pytest.approx(0.60) + assert row["alpha01_markov_loss_threshold"] == pytest.approx(0.70) + assert row["alpha01_markov_loss_cap"] == pytest.approx(0.70) + + +def test_body_selection_uses_exact_markov_threshold() -> None: + eligible = pd.DataFrame( + { + "semantic_policy_key": ["tail", "linear"], + "alpha01_realized_total_return": [220_000.0, 190_000.0], + "alpha01_markov_loss_threshold": [0.70, 0.34], + } + ) + + selected = frontier._body_candidate(eligible, markov_threshold=0.35) + + assert selected["semantic_policy_key"] == "linear" + + +def test_threshold_frontier_does_not_mislabel_tail_policy_as_under_half() -> None: + eligible = pd.DataFrame( + { + "semantic_policy_key": ["tail", "linear"], + "alpha01_realized_total_return": [220_000.0, 190_000.0], + "alpha01_markov_loss_threshold": [0.70, 0.45], + } + ) + + selected = frontier._best_under_threshold(eligible, 0.50) + + assert selected is not None + assert selected["semantic_policy_key"] == "linear" diff --git a/tests/test_scripts/test_build_pool93_point_baseline_audit.py b/tests/test_scripts/test_build_pool93_point_baseline_audit.py new file mode 100644 index 0000000..715c911 --- /dev/null +++ b/tests/test_scripts/test_build_pool93_point_baseline_audit.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from scripts.search.build_pool93_point_baseline_audit import ( + _comparison_table, + _format_comparison_tex, +) + + +def test_comparison_table_reports_return_cost_and_certificate_metrics() -> None: + point = { + "realized_return": 200.0, + "expected_return_net_point": 210.0, + "certificate": { + "n_funded": 10, + "weighted_outcome": 0.12, + "weighted_miscoverage": 0.11, + "gamma_cp": 0.50, + "endpoint_budget": 0.60, + "markov_loss_threshold": 0.70, + }, + } + selected = { + "realized_return": 180.0, + "expected_return_net_point": 170.0, + "certificate": { + "n_funded": 14, + "weighted_outcome": 0.04, + "weighted_miscoverage": 0.03, + "gamma_cp": 0.16, + "endpoint_budget": 0.25, + "markov_loss_threshold": 0.35, + }, + } + + table = _comparison_table(point, selected).set_index("policy") + + assert isinstance(table, pd.DataFrame) + assert table.loc["Point-PD two-stage LP", "return_cost_vs_point_pct"] == pytest.approx(0.0) + assert table.loc["Selected CRPTO", "return_cost_vs_point_pct"] == pytest.approx(10.0) + assert table.loc["Selected CRPTO", "Markov_threshold_alpha01"] == pytest.approx(0.35) + assert table.loc["Selected CRPTO", "weighted_default_rate"] == pytest.approx(0.04) + + tex = _format_comparison_tex(table.reset_index()) + assert "Policy & Realized return & Weighted default" in tex + assert "Selected CRPTO & \\$180.00 & 0.040000 & 0.160000 & 0.250000 & 0.350000" in tex + assert "expected\\_return\\_net\\_point" not in tex diff --git a/tests/test_scripts/test_compile_ijds_submission.py b/tests/test_scripts/test_compile_ijds_submission.py new file mode 100644 index 0000000..13c5480 --- /dev/null +++ b/tests/test_scripts/test_compile_ijds_submission.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from scripts.compile_ijds_submission import LatexScan, _windows_latexmk_script + + +def test_latex_scan_ok_property_flags_clean_build() -> None: + scan = LatexScan(pages=27, blg_warnings=(), log_failures=()) + + assert scan.ok + + +def test_latex_scan_ok_property_rejects_warnings_or_log_failures() -> None: + assert not LatexScan(pages=27, blg_warnings=("Warning--empty journal",), log_failures=()).ok + assert not LatexScan(pages=27, blg_warnings=(), log_failures=("undefined references",)).ok + + +def test_windows_latexmk_script_finds_tinytex_payload(tmp_path) -> None: + wrapper = tmp_path / "TinyTeX" / "bin" / "windows" / "latexmk.exe" + script = tmp_path / "TinyTeX" / "texmf-dist" / "scripts" / "latexmk" / "latexmk.pl" + wrapper.parent.mkdir(parents=True) + wrapper.touch() + script.parent.mkdir(parents=True) + script.touch() + + assert _windows_latexmk_script(wrapper) == script diff --git a/tests/test_scripts/test_export_pool93_policy_aware_frontier.py b/tests/test_scripts/test_export_pool93_policy_aware_frontier.py new file mode 100644 index 0000000..c548491 --- /dev/null +++ b/tests/test_scripts/test_export_pool93_policy_aware_frontier.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from scripts.search.export_pool93_policy_aware_frontier import ROLE_ORDER, build_table + + +def test_build_table_orders_roles_and_uses_exact_threshold() -> None: + rows = [] + for index, role in enumerate(reversed(ROLE_ORDER), start=1): + rows.append( + { + "role": role, + "run_label": "unit", + "local_candidate_id": index, + "family": "family", + "risk_tolerance": 0.17, + "policy_mode": "blended_uncertainty", + "gamma": 0.5, + "uncertainty_aversion": 0.1, + "return": 180_000.0 + index, + "return_floor_surplus": 9_000.0, + "Gamma_CP": 0.2, + "Gamma_residual": 0.1, + "V": 0.03, + "endpoint_budget": 0.24, + "endpoint_budget_upper": 0.24, + "Markov_threshold": 0.34, + "Markov_cap": 0.34, + "alpha_pass": "8/8", + "n_funded_mean": 300.0, + } + ) + + table = build_table({"rows": rows}) + + assert len(table) == len(ROLE_ORDER) + assert table.iloc[0]["role"] == "Minimum Markov-threshold endpoint" + assert table.iloc[-1]["role"] == "Max-return economic endpoint" + assert table["Markov_threshold_alpha01"].eq(0.34).all() + assert "Gamma_residual_alpha01" in table.columns diff --git a/tests/test_scripts/test_generate_conformal_intervals_cli.py b/tests/test_scripts/test_generate_conformal_intervals_cli.py index 572aa5d..b58f4dc 100644 --- a/tests/test_scripts/test_generate_conformal_intervals_cli.py +++ b/tests/test_scripts/test_generate_conformal_intervals_cli.py @@ -4,13 +4,19 @@ import scripts.generate_conformal_intervals as conformal_script from scripts.generate_conformal_intervals import ( + _apply_global_rebalance, + _apply_learned_floor_policy, + _build_conformal_artifact_tables, _build_tuning_split, + _can_use_temporal_segments, _parse_bool_tuple, _parse_float_tuple, _parse_int_tuple, _parse_str_tuple, _resolve_tuning_grid, + _select_alpha_95, _select_best_tuning_config, + _tuning_total_candidates, ) @@ -77,6 +83,27 @@ def test_resolve_tuning_grid_uses_current_defaults_for_empty_inputs() -> None: assert grid.scaled_scores_options == () +def test_tuning_total_candidates_counts_cartesian_grid() -> None: + grid = _resolve_tuning_grid( + partition="grade", + partition_candidates=("grade", "score_bin"), + partition_probability_sources=("raw",), + n_score_bins_candidates=(5, 10), + fallback_modes=("global",), + score_scale_families=("none",), + scaled_scores_options=(True, False), + ) + + assert ( + _tuning_total_candidates( + grid, + alpha_candidates_90=(0.10, 0.09), + min_group_sizes=(200, 500), + ) + == 32 + ) + + def test_build_tuning_split_materializes_fit_and_holdout( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -125,6 +152,145 @@ def fake_split_calibration_for_tuning(**kwargs): assert split.issue_test.isna().tolist() == [False, True] +def test_apply_global_rebalance_disabled_preserves_current_metrics() -> None: + y_int = conformal_script.np.array([[0.1, 0.4], [0.2, 0.7]]) + metrics = {"empirical_coverage": 0.9} + group_metrics = conformal_script.pd.DataFrame({"group": ["A"], "coverage": [0.9]}) + + result = _apply_global_rebalance( + enabled=False, + min_factor=0.75, + max_factor=1.05, + step=0.01, + y_int_tune_working=y_int, + y_pred_tune=conformal_script.np.array([0.25, 0.45]), + y_tune=conformal_script.pd.Series([0.0, 1.0]), + y_int_90=y_int, + y_pred_90=conformal_script.np.array([0.25, 0.45]), + y_eval_90=conformal_script.pd.Series([0.0, 1.0]), + group_tune=conformal_script.pd.Series(["A", "B"]), + eval_groups_90=conformal_script.pd.Series(["A", "B"]), + alpha_target_90=0.10, + target_coverage_90=0.90, + min_group_coverage_target=0.88, + metrics_90=metrics, + group_metrics_90=group_metrics, + ) + + assert result.y_intervals is y_int + assert result.metrics is metrics + assert result.group_metrics is group_metrics + assert result.factor == 1.0 + assert result.diagnostics == {"enabled": False, "applied": False} + + +def test_can_use_temporal_segments_requires_enabled_dates_and_matching_lengths() -> None: + issue_dates = conformal_script.pd.Series(["2020-01-01", "2020-02-01"]) + groups = conformal_script.pd.Series(["A", "B"]) + + assert _can_use_temporal_segments( + enabled=True, + issue_tune=issue_dates, + eval_issue=issue_dates, + group_tune=groups, + eval_groups=groups, + ) + assert not _can_use_temporal_segments( + enabled=False, + issue_tune=issue_dates, + eval_issue=issue_dates, + group_tune=groups, + eval_groups=groups, + ) + assert not _can_use_temporal_segments( + enabled=True, + issue_tune=conformal_script.pd.Series([None, None]), + eval_issue=issue_dates, + group_tune=groups, + eval_groups=groups, + ) + assert not _can_use_temporal_segments( + enabled=True, + issue_tune=issue_dates, + eval_issue=issue_dates, + group_tune=groups, + eval_groups=conformal_script.pd.Series(["A"]), + ) + + +def test_apply_learned_floor_policy_applies_group_temporal_and_global_factors() -> None: + adjusted = _apply_learned_floor_policy( + y_pred=conformal_script.np.array([0.5, 0.5]), + y_intervals=conformal_script.np.array([[0.4, 0.6], [0.4, 0.6]]), + groups=conformal_script.pd.Series(["A", "B"]), + group_multipliers={"A": 2.0}, + temporal_segments=conformal_script.pd.Series(["A|2020Q1", "B|2020Q1"]), + temporal_segment_multipliers={"B|2020Q1": 3.0}, + global_rebalance_factor=0.5, + ) + + conformal_script.np.testing.assert_allclose( + adjusted, + conformal_script.np.array([[0.4, 0.6], [0.35, 0.65]]), + ) + + +def test_build_conformal_artifact_tables_preserves_holdout_metadata() -> None: + tables = _build_conformal_artifact_tables( + y_eval_90=conformal_script.pd.Series([0.0, 1.0]), + y_pred_90=conformal_script.np.array([0.2, 0.8]), + y_int_90=conformal_script.np.array([[0.1, 0.4], [0.6, 0.9]]), + y_int_95=conformal_script.np.array([[0.0, 0.5], [0.5, 1.0]]), + eval_groups_90=conformal_script.pd.Series(["A", "B"]), + eval_temporal_segments=conformal_script.pd.Series(["A|2020Q1", "B|2020Q1"]), + evaluation_scope_key="holdout", + test_df=conformal_script.pd.DataFrame({"id": ["test-1"], "loan_amnt": [9000.0]}), + cal_df=conformal_script.pd.DataFrame( + { + "id": ["cal-0", "cal-1", "cal-2"], + "loan_amnt": [1000.0, 2000.0, 3000.0], + } + ), + idx_cal_tune=conformal_script.np.array([1, 2]), + group_metrics_90=conformal_script.pd.DataFrame( + { + "group": ["A", "B"], + "coverage": [0.9, 1.0], + "avg_width": [0.3, 0.3], + "median_width": [0.3, 0.3], + } + ), + group_metrics_95=conformal_script.pd.DataFrame( + { + "group": ["A", "B"], + "coverage": [0.95, 1.0], + "avg_width": [0.5, 0.5], + "median_width": [0.5, 0.5], + } + ), + coverage_floor_report=conformal_script.pd.DataFrame( + { + "group": ["A", "B"], + "coverage_before": [0.9, 1.0], + "coverage_after": [0.92, 1.0], + "multiplier": [1.02, 1.0], + "adjusted": [True, False], + } + ), + width_attr_rows=[{"dataset_scope": "holdout", "stage": "base_interval"}], + ) + + assert tables.intervals["_row_number"].tolist() == [0, 1] + assert tables.intervals["id"].tolist() == ["cal-1", "cal-2"] + assert tables.intervals["loan_amnt"].tolist() == [2000.0, 3000.0] + assert tables.intervals["temporal_segment"].tolist() == ["A|2020Q1", "B|2020Q1"] + assert tables.group_metrics["coverage_95"].tolist() == [0.95, 1.0] + assert tables.group_metrics["adjusted"].tolist() == [True, False] + assert tables.width_attribution.to_dict(orient="records") == [ + {"dataset_scope": "holdout", "stage": "base_interval"} + ] + + def test_build_tuning_split_rejects_empty_holdout(monkeypatch: pytest.MonkeyPatch) -> None: def fake_split_calibration_for_tuning(**kwargs): return conformal_script.np.array([0, 1]), conformal_script.np.array([], dtype=int) @@ -148,6 +314,48 @@ def fake_split_calibration_for_tuning(**kwargs): ) +def test_select_alpha_95_uses_holdout_gap_then_width(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_create_intervals(**kwargs): + alpha = float(kwargs["alpha"]) + return ( + conformal_script.np.array([0.2, 0.3]), + conformal_script.np.array([[0.0, alpha], [0.1, alpha + 0.1]]), + {}, + ) + + def fake_validate_coverage(*args, **kwargs): + alpha = float(kwargs["alpha"]) + return { + "coverage_gap": {0.04: 0.03, 0.05: 0.01, 0.06: 0.01}[alpha], + "avg_interval_width": {0.04: 0.20, 0.05: 0.18, 0.06: 0.16}[alpha], + } + + monkeypatch.setattr( + conformal_script, + "create_pd_intervals_mondrian_from_predictions", + fake_create_intervals, + ) + monkeypatch.setattr(conformal_script, "validate_coverage", fake_validate_coverage) + + selected_alpha = _select_alpha_95( + alpha_95=0.05, + alpha_candidates_95=(0.04, 0.05, 0.06), + interval_fit_pred=conformal_script.np.array([0.1, 0.2]), + interval_tune_pred=conformal_script.np.array([0.2, 0.3]), + y_cal_fit=conformal_script.pd.Series([0.0, 1.0]), + y_tune=conformal_script.pd.Series([0.0, 1.0]), + group_cal_fit_holdout=conformal_script.pd.Series(["A", "B"]), + group_tune=conformal_script.pd.Series(["A", "B"]), + best_cfg={ + "min_group_size": 200, + "scaled_scores": False, + "score_scale_family": "none", + }, + ) + + assert selected_alpha == 0.06 + + def test_select_best_tuning_config_materializes_promoted_config() -> None: rows = [ { diff --git a/tests/test_scripts/test_generate_crpto_figures.py b/tests/test_scripts/test_generate_crpto_figures.py new file mode 100644 index 0000000..9dc5c5b --- /dev/null +++ b/tests/test_scripts/test_generate_crpto_figures.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import pandas as pd + +from scripts.generate_crpto_figures import ( + PALETTE, + _alpha_annotation_offset, + _alpha_pareto_column_map, + _alpha_pareto_missing_columns, + _alpha_pareto_subframe, + _alpha_pareto_variant_styles, + _alpha_tick_labels, +) + + +def test_alpha_pareto_column_map_detects_semantic_columns() -> None: + df = pd.DataFrame( + columns=[ + "method_name", + "alpha_level", + "empirical_coverage", + "mean_width", + "n_eligible_loans", + ] + ) + + columns = _alpha_pareto_column_map(df) + + assert columns == { + "variant": "method_name", + "alpha": "alpha_level", + "coverage": "empirical_coverage", + "width": "mean_width", + "eligible": "n_eligible_loans", + } + assert _alpha_pareto_missing_columns(columns) == [] + + +def test_alpha_pareto_variant_styles_label_mondrian_and_global() -> None: + colors, labels = _alpha_pareto_variant_styles(["mondrian", "global"]) + + assert labels == { + "mondrian": "Mondrian CP", + "global": "Global Split-CP", + } + assert colors == { + "mondrian": PALETTE["blue"], + "global": PALETTE["orange"], + } + + +def test_alpha_pareto_subframe_sorts_alpha_and_formats_labels() -> None: + df = pd.DataFrame( + { + "variant": ["global", "mondrian", "global"], + "alpha": [0.2, 0.1, 0.05], + "coverage": [0.93, 0.91, 0.90], + } + ) + + sub = _alpha_pareto_subframe( + df, + variant_col="variant", + alpha_col="alpha", + variant="global", + ) + + assert list(sub["alpha"]) == [0.05, 0.2] + assert _alpha_tick_labels(sub["alpha"]) == ["0.05", "0.2"] + assert _alpha_annotation_offset(0, 2) == (4, 4) + assert _alpha_annotation_offset(1, 2) == (-24, -8) diff --git a/tests/test_scripts/test_generate_governance_status.py b/tests/test_scripts/test_generate_governance_status.py new file mode 100644 index 0000000..1ea0a13 --- /dev/null +++ b/tests/test_scripts/test_generate_governance_status.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pandas as pd + +from scripts.generate_governance_status import ( + GovernanceOutputPaths, + GovernanceThresholds, + _build_explanation_drift_report, + _build_governance_status, + _drift_breach_metrics, +) + + +def _test_thresholds() -> GovernanceThresholds: + return GovernanceThresholds( + psi_threshold=0.25, + ks_pvalue_min=0.01, + cvm_pvalue_min=0.01, + c2st_auc_max=0.60, + max_feature_breach_ratio=0.60, + c2st_max_rows=50_000, + score_psi_max=0.15, + auc_delta_max=0.05, + brier_increase_max=0.02, + calibration_gap_delta_max=0.02, + performance_max_rows=100_000, + min_rank_overlap_top10=0.60, + max_explanation_shap_psi=0.25, + min_reason_code_stability=0.55, + explanation_min_rows_per_slice=80, + psi_bins=10, + random_state=42, + ) + + +def _test_paths() -> GovernanceOutputPaths: + return GovernanceOutputPaths( + drift_path=Path("data/processed/drift_monitoring.parquet"), + status_path=Path("models/governance_status.json"), + explanation_drift_path=Path("data/processed/explanation_drift.parquet"), + fairness_status_path=Path("models/fairness_audit_status.json"), + fairness_frontier_path=Path("data/processed/fairness_threshold_frontier.parquet"), + challenger_report_path=Path("models/challenger_promotion_report.json"), + model_shift_status_path=Path("models/model_shift_status.json"), + ) + + +def test_build_explanation_drift_report_emits_overall_and_grade_rows() -> None: + rows: list[dict[str, object]] = [] + periods = ["2020Q1", "2020Q2", "2020Q3"] + for period in periods: + for grade in ["A", "B"]: + for idx in range(20): + rows.append( + { + "issue_quarter": period, + "grade": grade, + "pd_calibrated": 0.20, + "shap_dti": 0.30 + 0.001 * idx, + "shap_income": 0.10 + 0.001 * idx, + } + ) + shap_raw = pd.DataFrame(rows) + + report = _build_explanation_drift_report( + shap_raw, + primary_threshold=0.50, + min_rank_overlap_top10=0.50, + max_shap_psi=10.0, + min_reason_code_stability=0.50, + min_rows_per_slice=20, + ) + + assert set(report["segment_type"]) == {"overall", "grade"} + assert set(report["segment"]) == {"all", "A", "B"} + assert set(report["comparison_period"]) == {"2020Q3"} + assert report["passed_all"].all() + assert np.isfinite(report["max_shap_psi_top5"]).all() + + details = json.loads(str(report.loc[report["segment"] == "all", "feature_psi_details"].iloc[0])) + assert {row["feature"] for row in details} == {"dti", "income"} + + +def test_governance_status_helpers_preserve_public_contract() -> None: + thresholds = _test_thresholds() + drift_df = pd.DataFrame( + { + "pass_psi": [True, False], + "pass_ks": [True, True], + "pass_cvm": [True, False], + "psi": [0.05, 0.20], + "ks_pvalue": [0.50, 0.40], + "cvm_pvalue": [0.60, 0.30], + "feature": ["a", "b"], + } + ) + c2st = { + "c2st_auc": 0.55, + "materiality": "moderate", + "effective_driver_count": 1, + "top_drivers": ["a"], + "n_rows": 100, + } + performance = { + "score_psi": 0.10, + "auc_delta_train_to_test": 0.02, + "brier_increase_train_to_test": 0.01, + "calibration_gap_delta": 0.01, + } + metrics = _drift_breach_metrics(drift_df, c2st, performance, thresholds) + explanation_drift = pd.DataFrame( + { + "passed_all": [True], + "pass_reason_code_stability": [True], + "rank_overlap_top10": [0.80], + "max_shap_psi_top5": [0.10], + "reason_code_match_rate": [0.90], + } + ) + status = _build_governance_status( + config_path="configs/mrm_policy.yaml", + resolved_run_tag="test-run", + paths=_test_paths(), + thresholds=thresholds, + drift_df=drift_df, + explanation_drift=explanation_drift, + fairness_status={"overall_pass": True, "primary_threshold": 0.42}, + challenger_report={"challenger_promotable": True}, + metrics=metrics, + model_shift={"shift_type": "stable", "governance_posture": "monitor"}, + ) + + assert metrics["psi_breaches"] == 1 + assert metrics["pass_predictive_drift"] is True + assert status["overall_pass"] is True + assert status["checks"]["pass_explainability"] is True + assert status["summary"]["fairness_primary_threshold"] == 0.42 + assert status["summary"]["challenger_promotable"] is True + assert Path(status["artifacts"]["model_shift_status_path"]) == Path( + "models/model_shift_status.json" + ) + + +def test_build_explanation_drift_report_requires_enough_recent_rows() -> None: + shap_raw = pd.DataFrame( + { + "issue_quarter": ["2020Q1", "2020Q2"], + "pd_calibrated": [0.20, 0.20], + "shap_dti": [0.1, 0.2], + } + ) + + report = _build_explanation_drift_report( + shap_raw, + primary_threshold=0.50, + min_rank_overlap_top10=0.50, + max_shap_psi=10.0, + min_reason_code_stability=0.50, + min_rows_per_slice=20, + ) + + assert report.empty diff --git a/tests/test_scripts/test_optimize_portfolio_tradeoff.py b/tests/test_scripts/test_optimize_portfolio_tradeoff.py index bfcd56b..688add0 100644 --- a/tests/test_scripts/test_optimize_portfolio_tradeoff.py +++ b/tests/test_scripts/test_optimize_portfolio_tradeoff.py @@ -2,12 +2,109 @@ import numpy as np import pandas as pd +import pytest +from scripts.optimize_portfolio import _align_candidates_and_intervals from scripts.optimize_portfolio_tradeoff import ( + _align_loans_and_intervals, _build_policy_grid, _prepare_tradeoff_inputs, _select_champion_policy, + _solve_single, ) +from src.optimization import policy_evaluation + + +def test_portfolio_alignment_wrappers_share_strict_id_contract() -> None: + candidates = pd.DataFrame( + { + "id": ["b", "a", "c"], + "grade": ["B", "A", "C"], + "y_pred": [9.0, 8.0, 7.0], + } + ) + intervals = pd.DataFrame( + { + "id": ["a", "b", "c"], + "grade": ["score_q01", "score_q02", "score_q03"], + "y_pred": [0.1, 0.2, 0.3], + "pd_low_90": [0.05, 0.15, 0.25], + "pd_high_90": [0.15, 0.25, 0.35], + } + ) + + tradeoff_loans, tradeoff_intervals = _align_loans_and_intervals( + candidates, + intervals, + max_candidates=0, + random_state=42, + ) + optimizer_loans, pd_point, pd_low, pd_high = _align_candidates_and_intervals( + candidates, + intervals, + max_candidates=0, + random_state=42, + ) + + assert tradeoff_loans["id"].tolist() == ["b", "a", "c"] + assert tradeoff_intervals["grade"].tolist() == [ + "score_q02", + "score_q01", + "score_q03", + ] + pd.testing.assert_frame_equal(tradeoff_loans, optimizer_loans) + np.testing.assert_allclose(pd_point, [0.2, 0.1, 0.3]) + np.testing.assert_allclose(pd_low, [0.15, 0.05, 0.25]) + np.testing.assert_allclose(pd_high, [0.25, 0.15, 0.35]) + + +def test_nonrobust_solve_uses_point_pd_contract(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + + def _fake_optimize_portfolio_allocation(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return { + "allocation": {1: 0.5}, + "objective_value": 10.0, + "n_funded": 1, + "pd_cap_slack": 0.0, + "solver_status": "optimal", + } + + monkeypatch.setattr( + policy_evaluation, + "optimize_portfolio_allocation", + _fake_optimize_portfolio_allocation, + ) + loans = pd.DataFrame({"loan_amnt": [1000.0, 2000.0]}) + pd_point = np.array([0.10, 0.20]) + pd_high = np.array([0.40, 0.50]) + + result, allocation = _solve_single( + loans=loans, + pd_point=pd_point, + pd_low=np.array([0.05, 0.10]), + pd_high=pd_high, + lgd=np.array([0.45, 0.45]), + int_rates=np.array([0.10, 0.12]), + default_flag=np.array([0, 0]), + total_budget=1000.0, + max_concentration=1.0, + risk_tolerance=0.25, + robust=False, + uncertainty_aversion=0.0, + min_budget_utilization=0.0, + pd_cap_slack_penalty=0.0, + time_limit=10, + threads=1, + policy_mode="hard_worst_case", + gamma=1.0, + ) + + np.testing.assert_allclose(captured["pd_constraint_override"], pd_point) + np.testing.assert_allclose(allocation, [0.0, 0.5]) + assert result["policy_mode"] == "point_estimate" + assert result["gamma"] == 0.0 def test_build_policy_grid_preserves_tradeoff_frontier_contract() -> None: diff --git a/tests/test_scripts/test_pool93_local_refinement_grid.py b/tests/test_scripts/test_pool93_local_refinement_grid.py new file mode 100644 index 0000000..55c4087 --- /dev/null +++ b/tests/test_scripts/test_pool93_local_refinement_grid.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import hashlib + +import pandas as pd +import pytest + +from scripts.search.run_pool93_ijds_local_refinement import ( + DEFAULT_ALPHA_GRID, + _build_parser, + _claim_summary, + _generate_candidate_grid, + _manifest_payload, + _pending_refinement_tasks, + _resolve_paths, +) +from src.optimization.certificate_semantics import IJDS_DECLARED_ALPHA_GRID + +EXPECTED_PROFILE_FINGERPRINTS = { + "stage1": (1236, "4f4fa9791ad71b3901f0af5aa55ff616426700e3245e6d5fd4cd5e923086f6ad"), + "expanded": (7463, "a0bb03c5ee6491b2f9e1032e50c12c8aae6937a84206a989889acbb4212d371a"), + "claim_expanded": ( + 3659, + "8345545b20e93985462a84e92bf503417911386ab1aaf0194a657aab64d8d329", + ), + "claim_micro": (2949, "1e4083c8b8e200c5689566a06da1e18591a11b219b13ffea5c65d575dfb796bc"), + "claim_micro_ext": ( + 4407, + "3cc8a45b2ca0fde9a2f12cd6229b94cc7bf0119e3eeb7e742d6ce2f7bf601d08", + ), + "claim_bound_closure": ( + 1653, + "8d8ef58d92049809406049ff22b81831a9607c0203a2c401fb026826f2a9acee", + ), + "claim_bound_floor_closure": ( + 2343, + "9cb18594d1ec323cab23ffcdb4c96e481b2fb0798a18c4df73b796737e6b72c5", + ), + "claim_bound_terminal": ( + 37068, + "6d75ef0b7c083f9f60dfc834a50cb5da10873223dffe340665ef130e6c4c88ac", + ), +} + + +def test_pool93_default_alpha_grid_uses_shared_certificate_semantics() -> None: + assert list(IJDS_DECLARED_ALPHA_GRID) == DEFAULT_ALPHA_GRID + + +def _synthetic_anchor_rows() -> pd.DataFrame: + base_fields = { + "tail_focus_quantile": 1.0, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + } + return pd.DataFrame( + [ + { + **base_fields, + "candidate_rank": 96, + "risk_tolerance": 0.156, + "policy_mode": "tail_blended_uncertainty", + "gamma": 0.46, + "uncertainty_aversion": 0.125, + "delta_cap_quantile": 1.0, + }, + { + **base_fields, + "candidate_rank": 219, + "risk_tolerance": 0.171, + "policy_mode": "blended_uncertainty", + "gamma": 0.45, + "uncertainty_aversion": 0.1, + "delta_cap_quantile": 1.0, + }, + { + **base_fields, + "candidate_rank": 223, + "risk_tolerance": 0.173, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.40, + "uncertainty_aversion": 0.1, + "delta_cap_quantile": 0.95, + }, + ] + ) + + +def _semantic_fingerprint(frame: pd.DataFrame) -> str: + blob = "\n".join(frame["semantic_policy_key"].astype(str)).encode() + return hashlib.sha256(blob).hexdigest() + + +@pytest.mark.parametrize("profile", EXPECTED_PROFILE_FINGERPRINTS) +def test_pool93_local_refinement_grid_is_stable_by_profile(profile: str) -> None: + expected_rows, expected_sha = EXPECTED_PROFILE_FINGERPRINTS[profile] + + candidates = _generate_candidate_grid( + _synthetic_anchor_rows(), + profile=profile, + solver_backend="highspy", + ) + + assert len(candidates) == expected_rows + assert _semantic_fingerprint(candidates) == expected_sha + assert candidates["local_candidate_id"].tolist() == list(range(1, expected_rows + 1)) + assert candidates["semantic_policy_key"].is_unique + + +def test_pool93_local_refinement_rejects_unknown_profile() -> None: + with pytest.raises(ValueError, match="profile must be one of"): + _generate_candidate_grid( + _synthetic_anchor_rows(), + profile="broad_new_search", + solver_backend="highspy", + ) + + +def test_pool93_manifest_paths_and_pending_tasks_are_coherent(tmp_path) -> None: + args = _build_parser().parse_args( + [ + "--run-tag", + "unit/run", + "--output-dir", + str(tmp_path / "out"), + "--model-dir", + str(tmp_path / "model"), + "--source-bound-eval", + str(tmp_path / "source.parquet"), + "--source-selection", + str(tmp_path / "selection.json"), + ] + ) + paths = _resolve_paths(args, run_tag="unit_run") + candidates = pd.DataFrame({"local_candidate_id": [1, 2]}) + + pending = _pending_refinement_tasks( + candidates=candidates, + alpha_grid=[0.01, 0.03], + completed_keys={(1, 0.01)}, + ) + manifest = _manifest_payload( + args=args, + paths=paths, + run_tag="unit_run", + source_bound_eval=tmp_path / "source.parquet", + source_selection=tmp_path / "selection.json", + conformal_intervals_path="data/processed/conformal.parquet", + anchor_ranks=[96, 219, 223], + alpha_grid=[0.01, 0.03], + ) + + assert [(row["local_candidate_id"], alpha) for row, alpha in pending] == [ + (1, 0.03), + (2, 0.01), + (2, 0.03), + ] + assert manifest["candidates_path"] == str(paths.candidates_path) + assert manifest["claim_summary_path"] == str(paths.claim_summary_path) + assert manifest["run_tag"] == "unit_run" + + +def test_claim_summary_exposes_finite_grid_and_balanced_claim() -> None: + leaderboard = pd.DataFrame( + [ + { + "claim_rank": 1, + "local_candidate_id": 1, + "local_family": "endpoint", + "anchor_rank": 96, + "source_reason": "max_return", + "risk_tolerance": 0.18, + "policy_mode": "tail_blended_uncertainty", + "gamma": 0.40, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.05, + "alpha01_realized_total_return": 190000.0, + "alpha01_gamma_cp": 0.20, + "alpha01_weighted_miscoverage_V": 0.05, + "alpha01_weighted_pd_true": 0.11, + "alpha01_empirical_coverage_funded": 0.92, + "alpha01_exact_pass": True, + "all_alpha_pass": True, + "alpha_exact_pass_count": 2, + "alpha_exact_check_count": 2, + "alpha_mean_gamma_cp": 0.18, + "alpha_mean_weighted_miscoverage_V": 0.04, + "n_funded_mean": 50, + "allocator_backends": "highspy", + }, + { + "claim_rank": 2, + "local_candidate_id": 2, + "local_family": "body", + "anchor_rank": 219, + "source_reason": "balanced", + "risk_tolerance": 0.17, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.55, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.05, + "alpha01_realized_total_return": 185000.0, + "alpha01_gamma_cp": 0.10, + "alpha01_weighted_miscoverage_V": 0.03, + "alpha01_weighted_pd_true": 0.10, + "alpha01_empirical_coverage_funded": 0.94, + "alpha01_exact_pass": True, + "all_alpha_pass": True, + "alpha_exact_pass_count": 2, + "alpha_exact_check_count": 2, + "alpha_mean_gamma_cp": 0.11, + "alpha_mean_weighted_miscoverage_V": 0.03, + "n_funded_mean": 48, + "allocator_backends": "highspy", + }, + { + "claim_rank": 3, + "local_candidate_id": 3, + "local_family": "failed", + "anchor_rank": 223, + "source_reason": "not_all_alpha", + "risk_tolerance": 0.16, + "policy_mode": "blended_uncertainty", + "gamma": 0.60, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.10, + "alpha01_realized_total_return": 200000.0, + "alpha01_gamma_cp": 0.09, + "alpha01_weighted_miscoverage_V": 0.02, + "alpha01_weighted_pd_true": 0.09, + "alpha01_empirical_coverage_funded": 0.95, + "alpha01_exact_pass": True, + "all_alpha_pass": False, + "alpha_exact_pass_count": 1, + "alpha_exact_check_count": 2, + "alpha_mean_gamma_cp": 0.10, + "alpha_mean_weighted_miscoverage_V": 0.02, + "n_funded_mean": 45, + "allocator_backends": "highspy", + }, + ] + ) + bound_eval = pd.DataFrame( + { + "alpha": [0.01, 0.01, 0.03], + "all_bounds_hold": [True, False, True], + "violation": [0.0, 0.02, 0.0], + "gamma_cp": [0.1, 0.2, 0.15], + "weighted_miscoverage_V": [0.03, 0.05, 0.04], + } + ) + + summary = _claim_summary(leaderboard, bound_eval, alpha_grid=[0.03, 0.01, 0.01]) + + assert summary["finite_grid_policy"]["alpha_grid"] == [0.01, 0.03] + assert summary["n_policies"] == 3 + assert summary["n_all_alpha_passers"] == 2 + assert summary["n_all_alpha_passers_above_return_floor"] == 2 + assert summary["max_return_claim"]["local_candidate_id"] == 1 + assert summary["best_gamma_cp_return_floor_claim"]["local_candidate_id"] == 2 + assert summary["best_weighted_miscoverage_return_floor_claim"]["local_candidate_id"] == 2 + assert summary["balanced_return_bound_claim"]["local_candidate_id"] == 2 + assert summary["by_family"]["failed"]["n_all_alpha_passers"] == 0 + assert summary["by_alpha"]["0.01"]["n_checks"] == 2 diff --git a/tests/test_scripts/test_retired_search_entrypoints.py b/tests/test_scripts/test_retired_search_entrypoints.py new file mode 100644 index 0000000..3a2b2b9 --- /dev/null +++ b/tests/test_scripts/test_retired_search_entrypoints.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from scripts.search import run_conformal_search, run_portfolio_search + + +def test_retired_conformal_search_entrypoint_is_actionable(capsys) -> None: + assert run_conformal_search.main([]) == 2 + + captured = capsys.readouterr() + assert "retired" in captured.err + assert "run_conformal_reopen_search.py" in captured.err + + +def test_retired_portfolio_search_entrypoint_is_actionable(capsys) -> None: + assert run_portfolio_search.main([]) == 2 + + captured = capsys.readouterr() + assert "retired" in captured.err + assert "run_pool93_ijds_local_refinement.py" in captured.err diff --git a/tests/test_scripts/test_run_comparison.py b/tests/test_scripts/test_run_comparison.py new file mode 100644 index 0000000..30c945b --- /dev/null +++ b/tests/test_scripts/test_run_comparison.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from scripts.run_comparison import _collect_status_metadata, _gate_ab_no_regression + + +def _ab_status(control: float, robust: float, **extra: object) -> dict[str, object]: + return { + "metrics_a": {"total_return": control}, + "metrics_b": {"total_return": robust}, + **extra, + } + + +def _status(run_tag: str) -> dict[str, str]: + return { + "schema_version": "test.1", + "generated_at_utc": "2026-07-08T20:00:00+00:00", + "run_tag": run_tag, + } + + +def test_collect_status_metadata_passes_when_core_statuses_match() -> None: + cur = { + "dvc_metrics_meta": _status("run-a"), + "pipeline_summary": _status("run-a"), + "conformal_status": _status("run-a"), + "fairness_status": _status("run-a"), + "governance_status": _status("run-a"), + "ab_simulation_status": _status("run-a"), + } + + metadata = _collect_status_metadata(cur, expected_run_tag="run-a") + + assert metadata["passed"] is True + assert metadata["all_have_metadata"] is True + assert metadata["run_tags_observed"] == ["run-a"] + assert metadata["mismatched_run_tag_artifacts"] == [] + assert len(metadata["critical_artifacts"]) == 6 + + +def test_collect_status_metadata_allows_causal_insights_only_mismatch() -> None: + cur = { + "dvc_metrics_meta": _status("run-a"), + "pipeline_summary": _status("run-a"), + "conformal_status": _status("run-a"), + "fairness_status": _status("run-a"), + "governance_status": _status("run-a"), + "ab_simulation_status": _status("run-a"), + "causal_effect_status": _status("older-causal-run"), + } + + metadata = _collect_status_metadata(cur, expected_run_tag="run-a") + + assert metadata["passed"] is True + assert metadata["run_tag_matches_expected"] is False + assert metadata["run_tag_matches_expected_operational"] is True + assert metadata["causal_only_mismatch"] is True + assert metadata["non_causal_mismatched_run_tag_artifacts"] == [] + + +def test_gate_ab_no_regression_passes_when_current_self_gate_passes() -> None: + base = {"ab_simulation_status": _ab_status(100.0, 120.0)} + cur = { + "ab_simulation_status": _ab_status( + 101.0, + 119.0, + no_regression={ + "passed": True, + "diff_total_return": 18.0, + "tolerance_total_return": 5.0, + }, + comparison={"p_value": 0.12, "significant": False}, + n_candidates_used=50, + ) + } + + gate = _gate_ab_no_regression(base, cur) + + assert gate.name == "ab_no_regression" + assert gate.passed is True + assert gate.details["checks"]["self_no_regression_ok"] is True + assert gate.details["warnings"]["control_vs_baseline_warning"] is False + assert gate.details["diagnostics"]["gate_mode"] == "no_regression" + + +def test_gate_ab_no_regression_allows_selective_ambiguity_cross_gate() -> None: + base = {"ab_simulation_status": _ab_status(100.0, 120.0)} + cur = { + "ab_simulation_status": _ab_status( + 100.0, + 80.0, + decision_scenario="selective_ambiguity_defer", + no_regression={"passed": False, "diff_total_return": -20.0}, + cross_scenario_gate={"passed": True}, + ) + } + + gate = _gate_ab_no_regression(base, cur) + + assert gate.passed is True + assert gate.details["checks"]["self_no_regression_ok"] is True + assert gate.details["checks"]["cross_scenario_gate_ok"] is True + assert gate.details["warnings"]["robust_vs_baseline_warning"] is True diff --git a/tests/test_scripts/test_run_conformal_reopen_search.py b/tests/test_scripts/test_run_conformal_reopen_search.py index d684f32..d1348b3 100644 --- a/tests/test_scripts/test_run_conformal_reopen_search.py +++ b/tests/test_scripts/test_run_conformal_reopen_search.py @@ -1,8 +1,23 @@ from __future__ import annotations +import json +from pathlib import Path +from typing import Any + import pandas as pd -from scripts.search.run_conformal_reopen_search import _phase2_top_designs +from scripts.search import run_conformal_reopen_search as reopen +from scripts.search.run_conformal_reopen_search import ( + Phase1ConfirmationResult, + _maybe_apply_phase2, + _phase1_from_resume, + _phase2_metric_blocked, + _phase2_run_reason, + _phase2_top_designs, + _rank_phase2_candidates, + _run_phase1_oot_confirmation, + _run_phase2_search, +) def _design(*, partition: str, alpha: float, width: float, rank: int) -> dict[str, object]: @@ -61,3 +76,322 @@ def test_phase2_top_designs_falls_back_to_inner_when_oot_empty() -> None: assert source == "phase1_inner_aggregate" assert top.iloc[0]["partition"] == "score_decile_mondrian" assert set(top["phase2_design_source"]) == {"phase1_inner_aggregate"} + + +def _passing_policy(*, width: float = 0.42) -> dict[str, object]: + return { + "overall_pass": True, + "strict_overall_pass": True, + "methodological_justification_pass": True, + "coverage_90": 0.90, + "avg_width_90": width, + "min_group_coverage_90": 0.89, + "warning_alerts": 0, + "total_alerts": 0, + } + + +def test_phase1_resume_result_preserves_source_paths_and_winner( + tmp_path: Path, + monkeypatch: object, +) -> None: + aggregate = pd.DataFrame( + [ + _design(partition="grade", alpha=0.10, width=0.40, rank=1), + _design(partition="score_decile_mondrian", alpha=0.09, width=0.41, rank=2), + ] + ) + source_paths = { + "inner_aggregate": tmp_path / "source_aggregate.parquet", + "inner_search": tmp_path / "source_inner.parquet", + } + aggregate.to_parquet(source_paths["inner_aggregate"], index=False) + + shortlist = aggregate.head(1).copy() + + def fake_paths(_run_tag: str) -> dict[str, Path]: + return source_paths + + def fake_shortlist( + *, source_run_tag: str, top_k_inner: int + ) -> tuple[pd.DataFrame, dict[str, Any]]: + assert source_run_tag == "source-run" + assert top_k_inner == 1 + return shortlist, {"source": "source-run"} + + monkeypatch.setattr(reopen, "_reopen_artifact_paths", fake_paths) + monkeypatch.setattr(reopen, "_build_resume_shortlist", fake_shortlist) + + result = _phase1_from_resume( + resume_from_run_tag="source-run", + top_k_inner=1, + output_paths={"phase1_shortlist": tmp_path / "shortlist.parquet"}, + ) + + assert result.aggregate_path == str(source_paths["inner_aggregate"]) + assert result.inner_search_path == str(source_paths["inner_search"]) + assert result.inner_search_winner["partition"] == "grade" + assert result.resume_meta == {"source": "source-run"} + assert (tmp_path / "shortlist.parquet").exists() + + +def test_phase1_oot_confirmation_writes_ranked_candidate_frame( + tmp_path: Path, + monkeypatch: object, +) -> None: + shortlist = pd.DataFrame( + [ + _design(partition="grade", alpha=0.10, width=0.50, rank=1), + _design(partition="score_decile_mondrian", alpha=0.09, width=0.42, rank=2), + ] + ) + + def fake_run_candidate(**kwargs: Any) -> dict[str, object]: + rank = int(kwargs["rank"]) + return { + "namespace": f"candidate-{rank}", + "policy_status": _passing_policy(width=0.50 if rank == 1 else 0.42), + "set_status": {"summary": {"set_coverage": 0.91, "singleton_rate": 0.73}}, + "selection_status": {"promotion_pass": rank == 2, "selected_variant": "variant"}, + } + + monkeypatch.setattr(reopen, "_run_phase1_oot_candidate", fake_run_candidate) + + result = _run_phase1_oot_confirmation( + run_tag="test-run", + env={}, + shortlist=shortlist, + output_paths={"phase1_final_candidates": tmp_path / "phase1_final.parquet"}, + alpha_candidates_95=[0.05], + partition_candidates=["grade"], + partition_probability_sources=["calibrated"], + n_score_bins_candidates=[10], + fallback_modes=["grade_then_global"], + score_scale_families=["none"], + calibration_fractions=[1.0], + sidecar_cfg={}, + validation_cfg={}, + ) + + assert result.best_namespace == "candidate-2" + assert result.final_namespace == "candidate-2" + assert result.final_decision == "promotable_for_followup" + assert result.frame.iloc[0]["avg_width_90"] == 0.42 + assert (tmp_path / "phase1_final.parquet").exists() + + +def test_maybe_apply_phase2_respects_phase1_only() -> None: + phase1 = Phase1ConfirmationResult( + candidates=[], + frame=pd.DataFrame([{"namespace": "phase1"}]), + best_namespace="phase1", + final_policy=_passing_policy(), + final_sets={"summary": {"set_coverage": 0.91}}, + final_decision="promotable_for_followup", + final_namespace="phase1", + ) + + result = _maybe_apply_phase2( + run_tag="run", + upstream_run_tag="upstream", + env={}, + aggregated=pd.DataFrame(), + phase1=phase1, + phase1_only=True, + force_phase2=True, + alpha_candidates_95=[], + tuning_holdout_ratios=[], + inner_random_states=[], + partition_candidates=[], + partition_probability_sources=[], + n_score_bins_candidates=[], + fallback_modes=[], + score_scale_families=[], + calibration_fractions=[], + phase2_cfg={"enabled": True}, + sidecar_cfg={}, + validation_cfg={}, + ) + + assert result.final_namespace == "phase1" + assert result.final_decision == "promotable_for_followup" + assert result.phase2_summary is None + assert _phase2_run_reason(force_phase2=True, phase2_always_evaluate=True) == "forced" + + +def test_phase2_metric_gate_and_ranking_are_explicit() -> None: + assert _phase2_metric_blocked( + calibration_metrics={"ece": 0.12, "brier_score": 0.21}, + baseline_metrics={"ece": 0.10, "brier_score": 0.20}, + max_metric_degradation={"ece": 0.01, "brier_score": 0.02}, + ) + assert not _phase2_metric_blocked( + calibration_metrics={"ece": 0.105}, + baseline_metrics={"ece": 0.10}, + max_metric_degradation={"ece": 0.01}, + ) + + ranked = _rank_phase2_candidates( + pd.DataFrame( + [ + { + "artifact_namespace": "wide", + "calibrator_method": "platt", + "holdout_coverage": 0.91, + "holdout_width": 0.30, + "calibrator_ece": 0.01, + "calibrator_adaptive_ece": 0.01, + "calibrator_brier": 0.10, + "calibrator_phi_brier": 0.20, + "selection_rank": 1, + }, + { + "artifact_namespace": "centered", + "calibrator_method": "isotonic", + "holdout_coverage": 0.90, + "holdout_width": 0.50, + "calibrator_ece": 0.02, + "calibrator_adaptive_ece": 0.02, + "calibrator_brier": 0.10, + "calibrator_phi_brier": 0.10, + "selection_rank": 2, + }, + ] + ) + ) + + assert ranked.iloc[0]["artifact_namespace"] == "centered" + + +def _phase2_paths(tmp_path: Path) -> dict[str, Path]: + data_dir = tmp_path / "data" + models_dir = tmp_path / "models" + data_dir.mkdir() + models_dir.mkdir() + return { + "data_dir": data_dir, + "models_dir": models_dir, + "phase2_search": data_dir / "phase2.parquet", + "phase2_progress": models_dir / "phase2_progress.json", + } + + +def test_run_phase2_search_records_metric_gate_skip( + tmp_path: Path, + monkeypatch: Any, +) -> None: + paths = _phase2_paths(tmp_path) + aggregate = pd.DataFrame([_design(partition="grade", alpha=0.10, width=0.40, rank=1)]) + + def fake_fit_calibrator( + *, method: str, output_path: Path, upstream_run_tag: str + ) -> tuple[str, dict[str, float]]: + del output_path, upstream_run_tag + if method == "venn_abers": + return method, {"ece": 0.10} + return method, {"ece": 0.20} + + monkeypatch.setattr(reopen, "_reopen_artifact_paths", lambda _run_tag: paths) + monkeypatch.setattr(reopen, "_fit_calibrator", fake_fit_calibrator) + + decision, policy, sets, summary = _run_phase2_search( + run_tag="phase2-skip", + upstream_run_tag="upstream", + env={}, + aggregated=aggregate, + phase1_candidates_frame=pd.DataFrame(), + alpha_candidates_95=[0.05], + tuning_holdout_ratios=[0.2], + inner_random_states=[42], + partition_candidates=["grade"], + partition_probability_sources=["calibrated"], + n_score_bins_candidates=[10], + fallback_modes=["grade_then_global"], + score_scale_families=["none"], + calibration_fractions=[1.0], + phase2_cfg={"calibrators": ["platt"], "max_metric_degradation": {"ece": 0.01}}, + sidecar_cfg={}, + validation_cfg={}, + ) + + progress = json.loads(paths["phase2_progress"].read_text(encoding="utf-8")) + assert decision == "policy_review_candidate" + assert policy == {} + assert sets == {} + assert summary is not None + assert summary["status"] == "no_noninferior_calibrator_candidate" + assert progress["skipped"][0]["reason"] == "metric_degradation_gate" + assert paths["phase2_search"].exists() + + +def test_run_phase2_search_ranks_and_confirms_best_candidate( + tmp_path: Path, + monkeypatch: Any, +) -> None: + paths = _phase2_paths(tmp_path) + aggregate = pd.DataFrame( + [ + _design(partition="grade", alpha=0.10, width=0.50, rank=1), + _design(partition="score_decile_mondrian", alpha=0.09, width=0.42, rank=2), + ] + ) + + def fake_fit_calibrator( + *, method: str, output_path: Path, upstream_run_tag: str + ) -> tuple[str, dict[str, float]]: + del output_path, upstream_run_tag + return method, {"ece": 0.02, "adaptive_ece": 0.03, "brier_score": 0.10} + + def fake_resolve_run_paths(namespace: str) -> dict[str, Path]: + return {"results": tmp_path / f"{namespace}.pkl"} + + def fake_load_pickle(path: Path) -> dict[str, dict[str, float]]: + if "rank-2" in path.name: + return {"metrics_90": {"empirical_coverage": 0.90, "avg_interval_width": 0.44}} + return {"metrics_90": {"empirical_coverage": 0.88, "avg_interval_width": 0.30}} + + def fake_final_candidate(**kwargs: Any) -> dict[str, object]: + assert kwargs["phase_prefix"] == "phase2" + assert kwargs["design"]["selection_rank"] == 2 + assert str(kwargs["calibrator_override_path"]).endswith("platt.pkl") + return { + "namespace": "phase2-final", + "policy_status": _passing_policy(width=0.44), + "set_status": {"summary": {"set_coverage": 0.91}}, + } + + monkeypatch.setattr(reopen, "_reopen_artifact_paths", lambda _run_tag: paths) + monkeypatch.setattr(reopen, "_fit_calibrator", fake_fit_calibrator) + monkeypatch.setattr(reopen, "_run_python", lambda *_args, **_kwargs: None) + monkeypatch.setattr(reopen, "_resolve_run_paths", fake_resolve_run_paths) + monkeypatch.setattr(reopen, "_load_pickle", fake_load_pickle) + monkeypatch.setattr(reopen, "_run_phase1_oot_candidate", fake_final_candidate) + + decision, policy, sets, summary = _run_phase2_search( + run_tag="phase2-success", + upstream_run_tag="upstream", + env={}, + aggregated=aggregate, + phase1_candidates_frame=pd.DataFrame(), + alpha_candidates_95=[0.05], + tuning_holdout_ratios=[0.2], + inner_random_states=[42], + partition_candidates=["grade"], + partition_probability_sources=["calibrated"], + n_score_bins_candidates=[10], + fallback_modes=["grade_then_global"], + score_scale_families=["none"], + calibration_fractions=[1.0], + phase2_cfg={"calibrators": ["platt"], "max_metric_degradation": {}}, + sidecar_cfg={}, + validation_cfg={}, + ) + + phase2_search = pd.read_parquet(paths["phase2_search"]) + assert decision == "promotable_for_followup" + assert policy["avg_width_90"] == 0.44 + assert sets["summary"]["set_coverage"] == 0.91 + assert summary is not None + assert summary["final_namespace"] == "phase2-final" + assert summary["best_candidate"]["selection_rank"] == 2 + assert phase2_search.iloc[0]["selection_rank"] == 2 diff --git a/tests/test_scripts/test_run_crpto_vs_spo_stability.py b/tests/test_scripts/test_run_crpto_vs_spo_stability.py index a415e66..7f483c0 100644 --- a/tests/test_scripts/test_run_crpto_vs_spo_stability.py +++ b/tests/test_scripts/test_run_crpto_vs_spo_stability.py @@ -1,8 +1,15 @@ from __future__ import annotations import json +from argparse import Namespace from pathlib import Path +import numpy as np +import pandas as pd +import pytest + +from scripts import run_crpto_vs_spo_stability as stability_mod + def test_crpto_vs_spo_stability_artifacts_exist() -> None: status_path = Path("data/processed/crpto_vs_spo_stability.json") @@ -10,3 +17,59 @@ def test_crpto_vs_spo_stability_artifacts_exist() -> None: status = json.loads(status_path.read_text(encoding="utf-8")) assert status.get("schema_version") assert Path("reports/crpto/figures/crpto_fig11_crpto_stability.png").exists() + + +def test_period_sample_seed_is_stable_and_distinct_by_period() -> None: + seed = 42 + + assert stability_mod._period_sample_seed(seed, "2018H1") == 100_042 + assert stability_mod._period_sample_seed(seed, "2018H1") == 100_042 + assert stability_mod._period_sample_seed(seed, "2020") == 500_042 + + +def test_detail_rows_and_summary_payload_preserve_period_contract() -> None: + periods = list(stability_mod.PERIODS) + test = pd.DataFrame({"default_flag": [0, 1, 0, 1, 0]}) + period_masks = { + period: np.array([idx == pos for idx in range(len(periods))]) + for pos, period in enumerate(periods) + } + regrets = stability_mod._init_period_regrets() + regrets["2018H1"]["two_stage"] = [2.0, 4.0] + regrets["2018H1"]["spo_plus"] = [1.0, 2.0] + regrets["2018H1"]["conformal_robust"] = [3.0, float("nan")] + coverage = { + "2018H1": { + "coverage_90": 0.91, + "coverage_95": 0.96, + "avg_width_90": 0.42, + "min_grade_coverage_90": 0.89, + } + } + + rows = stability_mod._detail_rows( + test=test, + period_masks=period_masks, + per_period_regrets=regrets, + period_coverage=coverage, + ) + summary = stability_mod._summary_payload( + run_tag="run-test", + args=Namespace(n_items=50, budget=15, n_train=800, epochs=50, seeds=2), + n_features=3, + feature_names=["a", "b", "c"], + rows=rows, + per_period_regrets=regrets, + total_time=12.34, + ) + + first = rows[0] + assert first["period"] == "2018H1" + assert first["two_stage_mean_regret"] == 3.0 + assert first["spo_plus_mean_regret"] == 1.5 + assert first["conformal_robust_mean_regret"] == 3.0 + assert first["spo_improvement_pct"] == pytest.approx(49.999999983333336) + assert summary["config"]["n_features"] == 3 + assert summary["per_period"]["2018H1"]["coverage_90"] == 0.91 + assert summary["per_period"]["2018H1"]["spo_improvement_vs_ts_pct"] == 50.0 + assert summary["stability_summary"]["coverage_always_above_target"] is True diff --git a/tests/test_scripts/test_run_fairness_audit.py b/tests/test_scripts/test_run_fairness_audit.py index 8ab46d3..f3322b5 100644 --- a/tests/test_scripts/test_run_fairness_audit.py +++ b/tests/test_scripts/test_run_fairness_audit.py @@ -4,6 +4,7 @@ import json +import numpy as np import pandas as pd import pytest import yaml @@ -184,3 +185,57 @@ def test_run_fairness_auto_selects_threshold_and_writes_decision_policy(tmp_path assert status["prediction_threshold_source"] == "decision_policy_artifact_auto_selected" assert status["decision_policy"]["path"].endswith("fairness_decision_policy.json") assert decision_policy["global_threshold"] in [0.35, 0.4, 0.45, 0.5] + + +def test_shap_helpers_detect_categorical_columns_and_fill_missing_values() -> None: + class DummyModel: + def __init__(self) -> None: + self.feature_names_ = ["grade", "purpose", "rate"] + + def get_cat_feature_indices(self) -> list[int]: + return [0] + + frame = pd.DataFrame( + { + "grade": ["A", None, "B"], + "purpose": ["debt", "car", None], + "rate": [0.1, np.nan, 0.3], + } + ) + + cat_names = fairness_mod._catboost_cat_feature_names(DummyModel(), frame) + prepared = fairness_mod._prepare_catboost_shap_frame(frame, cat_names) + + assert cat_names == ["grade", "purpose"] + assert prepared.loc[1, "grade"] == "missing" + assert prepared.loc[2, "purpose"] == "missing" + assert prepared.loc[1, "rate"] == 0.0 + + +def test_shap_attribute_result_reports_group_drivers_and_pairwise_diffs() -> None: + shap_matrix = np.vstack( + [ + np.tile([3.0, 1.0, 0.5], (10, 1)), + np.tile([1.0, 4.0, 0.5], (10, 1)), + ] + ) + sample_idx = np.arange(20) + groups = { + "home_ownership": np.array(["A"] * 10 + ["B"] * 10), + "home_ownership__x__grade": np.array(["skip"] * 20), + } + + results = fairness_mod._shap_attribute_results( + groups, + sample_idx=sample_idx, + shap_matrix=shap_matrix, + feature_names=["dti", "income", "grade"], + ) + + assert len(results) == 1 + result = results[0] + assert result["attribute"] == "home_ownership" + assert result["groups_analyzed"] == ["A", "B"] + assert result["top5_per_group"]["A"][0]["feature"] == "dti" + assert result["top5_per_group"]["B"][0]["feature"] == "income" + assert result["pairwise_feature_diffs"][0]["top_driving_features"][0]["feature"] == "income" diff --git a/tests/test_scripts/test_run_portfolio_bound_aware_search.py b/tests/test_scripts/test_run_portfolio_bound_aware_search.py index 25d2020..6df7fb5 100644 --- a/tests/test_scripts/test_run_portfolio_bound_aware_search.py +++ b/tests/test_scripts/test_run_portfolio_bound_aware_search.py @@ -1,13 +1,25 @@ from __future__ import annotations import pandas as pd +import pytest from scripts.search.run_portfolio_bound_aware_search import ( _aggregate_exact_results, + _budget_profiles, + _build_grid_spec, + _build_parser, _build_stratified_shortlist, _policy_semantic_key, + _resolve_run_paths, + _sanitize_run_label, + _search_space_payload, + _selection_context_payload, _targeted_policy_grid, ) +from src.optimization.certificate_semantics import ( + IJDS_DECLARED_ALPHA_GRID, + IJDS_DECLARED_ALPHA_GRID_CSV, +) def _frontier_row( @@ -227,3 +239,113 @@ def test_targeted_policy_grid_includes_segment_tail_families() -> None: "segment_tail_blended_uncertainty", "segment_relative_tail_blended_uncertainty", } + assert grid == [ + ("blended_uncertainty", 0.5, 1.0, 1.0), + ("capped_blended_uncertainty", 0.5, 0.75, 1.0), + ("tail_blended_uncertainty", 0.5, 1.0, 0.9), + ("segment_tail_blended_uncertainty", 0.5, 1.0, 0.9), + ("segment_relative_tail_blended_uncertainty", 0.5, 1.0, 0.9), + ] + + +def test_budget_profiles_are_explicit_and_reject_unknown_tokens() -> None: + profiles = _budget_profiles("free,floored") + + assert [profile["name"] for profile in profiles] == ["free_budget", "floored_budget"] + assert profiles[0]["min_budget_utilization"] == 0.0 + assert profiles[1]["pd_cap_slack_penalty"] == 1.5 + with pytest.raises(ValueError, match="Unsupported budget profile"): + _budget_profiles("free,aggressive") + + +def test_bound_aware_default_uses_declared_ijds_alpha_grid() -> None: + args = _build_parser().parse_args(["--conformal-intervals-path", "intervals.parquet"]) + + assert args.alpha_grid == IJDS_DECLARED_ALPHA_GRID_CSV + + +def test_grid_spec_separates_proxy_and_exact_sampling_contracts() -> None: + args = _build_parser().parse_args( + [ + "--conformal-intervals-path", + "intervals.parquet", + "--risk-grid", + "0.16,0.17", + "--random-states", + "42,52", + "--exact-random-states", + "62,72,82", + "--max-candidates", + "5000", + "--exact-max-candidates", + "0", + ] + ) + + grid = _build_grid_spec(args) + + assert grid.random_states == [42, 52] + assert grid.exact_random_states == [62, 72, 82] + assert grid.exact_max_candidates == 0 + assert grid.alpha_grid == list(IJDS_DECLARED_ALPHA_GRID) + assert grid.bound_total_checks(shortlist_size=5) == 5 * 8 * 3 + assert grid.frontier_total_units == 2 * 2 * (1 + grid.policy_grid_count) + + +def test_selection_context_payload_keeps_frontier_and_exact_paths_together(tmp_path) -> None: + args = _build_parser().parse_args( + [ + "--conformal-intervals-path", + str(tmp_path / "intervals.parquet"), + "--run-label", + "unit/run", + "--output-dir", + str(tmp_path / "out"), + "--model-dir", + str(tmp_path / "model"), + "--random-states", + "42,52", + "--exact-random-states", + "42", + "--policy-modes", + "blended_uncertainty", + ] + ) + run_label = _sanitize_run_label(args.run_label) + paths = _resolve_run_paths(args, run_label=run_label) + search_space = _search_space_payload( + args=args, + risk_values=[0.16], + aversion_values=[0.0], + gamma_values=[0.5], + delta_cap_quantiles=[1.0], + tail_focus_quantiles=[1.0], + budget_profiles=_budget_profiles(args.budget_profiles), + alpha_grid=[0.01], + random_states=[42, 52], + exact_random_states=[42], + exact_max_candidates=0, + policy_modes=["blended_uncertainty"], + cuopt_parameters={"method": "concurrent"}, + incumbent_risk_neighbors=[0.16], + incumbent_gamma_neighbors=[0.5], + incumbent_policy_modes=["blended_uncertainty"], + ) + + context = _selection_context_payload( + args=args, + paths=paths, + run_label=run_label, + search_space=search_space, + exact_max_candidates=0, + random_states=[42, 52], + exact_random_states=[42], + alpha_grid=[0.01], + ) + + assert run_label == "unit_run" + assert context["search_space"]["random_states"] == [42, 52] + assert context["search_space"]["exact_random_states"] == [42] + assert context["shortlist_path"] == str(paths.shortlist_path) + assert context["selection_path"] == str(paths.selection_path) + assert context["resource_snapshot_path"] == str(paths.resource_path) diff --git a/tests/test_scripts/test_run_portfolio_bound_exact_eval.py b/tests/test_scripts/test_run_portfolio_bound_exact_eval.py index 2f492d7..203aa63 100644 --- a/tests/test_scripts/test_run_portfolio_bound_exact_eval.py +++ b/tests/test_scripts/test_run_portfolio_bound_exact_eval.py @@ -10,9 +10,12 @@ _context_exact_threads, _context_max_candidates, _context_random_states, + _exact_eval_plan, _load_completed_bound_eval, _load_partial_bound_eval, _repo_relative, + _resume_exact_rows, + _search_space_payload, _shortlist_exact_path, _validate_alpha_grid_supported, ) @@ -84,6 +87,28 @@ def test_exact_context_overrides_proxy_sampling() -> None: assert _context_random_states(context) == [42, 52, 62] +def test_exact_plan_dedupes_full_universe_random_states(monkeypatch) -> None: + monkeypatch.delenv("EXACT_THREADS", raising=False) + context = { + "alpha_grid": [0.01, 0.03], + "max_candidates": 100000, + "exact_max_candidates": 0, + "random_states": [42], + "requested_exact_random_states": "42,52,62", + "exact_checkpoint_every": 7, + "exact_threads": 3, + } + + plan = _exact_eval_plan(context=context, shortlist_rows=5) + + assert plan.requested_random_states == [42, 52, 62] + assert plan.random_states == [42] + assert plan.full_universe_seed_deduped is True + assert plan.expected_checks == 10 + assert plan.exact_threads == 3 + assert plan.checkpoint_every == 7 + + def test_exact_threads_can_come_from_environment(monkeypatch) -> None: monkeypatch.setenv("EXACT_THREADS", "8") @@ -108,6 +133,34 @@ def test_repo_relative_keeps_artifact_paths_standalone() -> None: assert _repo_relative(path) == "models/portfolio_bound_aware/selection.json" +def test_resume_exact_rows_filters_out_irrelevant_seeds(tmp_path) -> None: + path = tmp_path / "bound_eval.parquet" + pd.DataFrame( + { + "candidate_rank": [1, 1, 2], + "eval_random_state": [42, 52, 42], + "alpha": [0.01, 0.01, 0.03], + "all_bounds_hold": [True, True, False], + } + ).to_parquet(path, index=False) + + rows, keys = _resume_exact_rows(bound_eval_path=path, random_states=[42]) + + assert len(rows) == 2 + assert keys == {(1, 42, 0.01), (2, 42, 0.03)} + + +def test_search_space_payload_records_requested_alpha_grid() -> None: + context = {"search_space": {"alpha_grid": [0.01], "mode": ["capped"]}} + + payload = _search_space_payload(context=context, alpha_grid=[0.01, 0.03]) + + assert payload["alpha_grid"] == [0.01, 0.03] + assert payload["requested_alpha_grid"] == [0.01] + assert payload["effective_alpha_grid"] == [0.01, 0.03] + assert payload["mode"] == ["capped"] + + def test_validate_alpha_grid_supported_accepts_sweep_values(tmp_path, monkeypatch) -> None: sweep_path = tmp_path / "data" / "processed" / "alpha_sweep_pareto_mondrian.parquet" sweep_path.parent.mkdir(parents=True) diff --git a/tests/test_scripts/test_run_regret_auditability_sandbox.py b/tests/test_scripts/test_run_regret_auditability_sandbox.py index 8161439..23b5bbb 100644 --- a/tests/test_scripts/test_run_regret_auditability_sandbox.py +++ b/tests/test_scripts/test_run_regret_auditability_sandbox.py @@ -6,9 +6,15 @@ import pytest import yaml +import scripts.search.run_regret_auditability_sandbox as sandbox from scripts.search.run_regret_auditability_sandbox import ( FEATURE_PROFILES, MONOTONIC_POLICIES, + PORTFOLIO_ALPHA_GRID, + PhaseCommand, + _pd_validation_policy, + _pending_commands_for_group, + _phase_command_groups, _rank_pd_candidate_rows, assert_safe_output_path, build_phase_commands, @@ -117,6 +123,69 @@ def test_resume_manifest_loading_roundtrip(tmp_path: Path) -> None: assert load_resume_manifest(tmp_path / "missing.json") == {} +def _command(name: str, phase: str, output: Path) -> PhaseCommand: + return PhaseCommand( + name=name, + phase=phase, + command=["python", "-c", "pass"], + outputs=[str(output)], + checkpoint=str(output), + env={}, + max_workers=1, + cpu_threads=1, + ) + + +def test_phase_command_groups_keep_consecutive_phase_batches(tmp_path: Path) -> None: + commands = [ + _command("a", "pd-smoke", tmp_path / "a"), + _command("b", "pd-smoke", tmp_path / "b"), + _command("c", "conformal", tmp_path / "c"), + _command("d", "pd-smoke", tmp_path / "d"), + ] + + groups = _phase_command_groups(commands) + + assert [[command.name for command in group] for group in groups] == [["a", "b"], ["c"], ["d"]] + + +def test_pending_commands_skip_completed_outputs_on_resume( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + completed_output = tmp_path / "done.txt" + completed_output.write_text("ok", encoding="utf-8") + pending_output = tmp_path / "todo.txt" + monkeypatch.setattr(sandbox, "_log_command_to_mlflow", lambda **_: None) + + pending, skipped = _pending_commands_for_group( + artifact_root=tmp_path, + log_path=tmp_path / "command_log.csv", + group=[ + _command("done", "pd-smoke", completed_output), + _command("todo", "pd-smoke", pending_output), + ], + resume=True, + ) + + assert skipped == 1 + assert [command.name for command in pending] == ["todo"] + assert "skipped_completed" in (tmp_path / "command_log.csv").read_text(encoding="utf-8") + + +def test_pd_validation_policy_scales_by_phase() -> None: + smoke_replay, smoke_walk_forward = _pd_validation_policy("pd-smoke") + broad_replay, broad_walk_forward = _pd_validation_policy("pd-broad") + refine_replay, refine_walk_forward = _pd_validation_policy("pd-refine") + + assert smoke_replay["top_k_trials"] == 1 + assert smoke_walk_forward is False + assert broad_replay["seeds"] == [42, 52, 62] + assert broad_walk_forward is True + assert refine_replay["top_k_trials"] == 30 + assert refine_walk_forward is True + + def test_pd_candidate_ranking_prefers_auc_then_calibration() -> None: ranked = _rank_pd_candidate_rows( [ @@ -172,6 +241,9 @@ def test_build_portfolio_command_uses_external_output_dirs(tmp_path: Path) -> No command = commands[0] assert "--output-dir" in command.command assert "--model-dir" in command.command + alpha_grid_index = command.command.index("--alpha-grid") + 1 + assert command.command[alpha_grid_index] == PORTFOLIO_ALPHA_GRID + assert PORTFOLIO_ALPHA_GRID == "0.01,0.03,0.05,0.07,0.10,0.12,0.15,0.20" assert all(str(tmp_path) in output for output in command.outputs) diff --git a/tests/test_scripts/test_run_ty_advisory.py b/tests/test_scripts/test_run_ty_advisory.py new file mode 100644 index 0000000..43db7b3 --- /dev/null +++ b/tests/test_scripts/test_run_ty_advisory.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from scripts.run_ty_advisory import TY_REQUIREMENT, build_ty_command, iter_python_files + + +def test_active_ty_scope_excludes_archived_optional_and_protected_paths() -> None: + files = set(iter_python_files(scope="active")) + + assert "scripts/generate_conformal_intervals.py" not in files + assert "scripts/train_pd_model.py" not in files + assert "scripts/run_spo_real.py" not in files + assert "src/optimization/cuopt_adapter.py" not in files + assert all(not path.startswith("scripts/archive/") for path in files) + assert "scripts/experiments/ijds_policy_support.py" in files + assert "scripts/experiments/run_ijds_calibration_selected_policy_challenger.py" in files + assert "scripts/experiments/run_ijds_exact_alpha_grid_challenger.py" in files + assert all( + not (path.startswith("scripts/search/run_") and path.endswith(".py")) for path in files + ) + + +def test_active_ty_scope_keeps_live_ijds_helpers() -> None: + files = set(iter_python_files(scope="active")) + + assert "scripts/compile_ijds_submission.py" in files + assert "scripts/build_ijds_calibration_selected_evidence.py" in files + assert "scripts/search/build_pool93_body_allocation_audit.py" in files + assert "src/models/conformal_alpha_grid.py" in files + assert "src/optimization/policy_selection.py" in files + assert "src/optimization/portfolio_model.py" in files + + +def test_full_ty_scope_keeps_every_python_file_under_src_and_scripts() -> None: + files = set(iter_python_files(scope="full")) + + assert "scripts/generate_conformal_intervals.py" in files + assert "scripts/train_pd_model.py" in files + assert "src/optimization/cuopt_adapter.py" in files + + +def test_ty_command_pins_version_and_keeps_daily_scope_advisory() -> None: + command = build_ty_command( + uvx="uvx", + files=["src/example.py"], + fail_on_diagnostics=False, + ) + + assert command[:4] == ["uvx", "--from", TY_REQUIREMENT, "ty"] + assert "--exit-zero" in command + assert command[-1] == "src/example.py" + + +def test_ty_command_can_block_the_submission_gate() -> None: + command = build_ty_command( + uvx="uvx", + files=["src/example.py"], + fail_on_diagnostics=True, + ) + + assert "--exit-zero" not in command diff --git a/tests/test_scripts/test_simulate_ab_test.py b/tests/test_scripts/test_simulate_ab_test.py new file mode 100644 index 0000000..93907e1 --- /dev/null +++ b/tests/test_scripts/test_simulate_ab_test.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import json + +import pandas as pd +import pytest + +from scripts.simulate_ab_test import _resolve_robust_policy + + +def test_resolve_robust_policy_uses_guardrail_champion_priority(tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + model_dir = tmp_path / "models" + model_dir.mkdir() + (model_dir / "champion_portfolio_policy.json").write_text( + json.dumps( + { + "selected_policy": {"risk_tolerance": 0.10, "gamma": 0.1}, + "selected_policy_balanced_robustness": { + "risk_tolerance": 0.11, + "policy_mode": "balanced", + "gamma": 0.2, + }, + "selected_policy_guardrail_robustness": { + "risk_tolerance": 0.12, + "policy_mode": "guardrail", + "gamma": 0.3, + }, + } + ), + encoding="utf-8", + ) + + policy = _resolve_robust_policy( + max_portfolio_pd=0.20, + policy_selector="guardrail_robustness", + ) + + assert policy["source"] == "champion_policy_artifact::guardrail_robustness" + assert policy["policy_mode"] == "guardrail" + assert policy["risk_tolerance"] == 0.12 + assert policy["gamma"] == 0.3 + + +def test_resolve_robust_policy_uses_summary_when_champion_missing(tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + summary_path = tmp_path / "data" / "processed" / "portfolio_robustness_summary.parquet" + summary_path.parent.mkdir(parents=True) + pd.DataFrame( + [ + { + "risk_tolerance": 0.15, + "best_robust_lambda": 0.2, + "best_robust_min_budget_utilization": 0.90, + "best_robust_pd_cap_slack_penalty": 1.0, + "best_robust_return": 100.0, + "best_robust_policy_mode": "hard_worst_case", + "best_robust_gamma": 0.8, + "best_robust_delta_cap_quantile": 0.95, + }, + { + "risk_tolerance": 0.18, + "best_robust_lambda": 0.4, + "best_robust_min_budget_utilization": 0.91, + "best_robust_pd_cap_slack_penalty": 2.0, + "best_robust_return": 200.0, + "best_robust_policy_mode": "blended_uncertainty", + "best_robust_gamma": 0.6, + "best_robust_delta_cap_quantile": 0.90, + }, + ] + ).to_parquet(summary_path, index=False) + + policy = _resolve_robust_policy(max_portfolio_pd=0.17) + + assert policy["source"] == "portfolio_robustness_summary" + assert policy["risk_tolerance"] == 0.15 + assert policy["uncertainty_aversion"] == 0.2 + assert policy["gamma"] == 0.8 + + +def test_resolve_robust_policy_explicit_champion_only_requires_artifact( + tmp_path, monkeypatch +) -> None: + monkeypatch.chdir(tmp_path) + + with pytest.raises(FileNotFoundError): + _resolve_robust_policy( + max_portfolio_pd=0.20, + policy_selector="explicit_champion_only", + ) diff --git a/tests/test_scripts/test_train_pd_model_config_overrides.py b/tests/test_scripts/test_train_pd_model_config_overrides.py index 484a06b..93799aa 100644 --- a/tests/test_scripts/test_train_pd_model_config_overrides.py +++ b/tests/test_scripts/test_train_pd_model_config_overrides.py @@ -1,5 +1,9 @@ from __future__ import annotations +import json +import sys +import types + import numpy as np import pytest @@ -7,11 +11,14 @@ from scripts.train_pd_model import ( _apply_cli_overrides, _apply_pd_replay_manifest, + _gate_tier, _load_training_splits, _normalize_sample_size, _prepare_training_inputs, + _replay_selection_policy, _sample_training_splits, _select_calibration_from_backtest, + _summarize_replayed_trials, ) @@ -219,6 +226,45 @@ def fake_resolve_feature_sets(*args, **kwargs): assert replay_stable_core_meta == {"enabled": False, "excluded_features": []} +def test_replay_trial_summary_prioritizes_gate_then_ece_then_auc() -> None: + rows = [ + { + "trial_number": 1, + "validation_auc": 0.80, + "validation_brier": 0.20, + "validation_ece": 0.05, + "gate_attrs_present": True, + "gate_all_pass": False, + }, + { + "trial_number": 2, + "validation_auc": 0.79, + "validation_brier": 0.21, + "validation_ece": 0.06, + "gate_attrs_present": True, + "gate_all_pass": True, + }, + { + "trial_number": 3, + "validation_auc": 0.90, + "validation_brier": 0.19, + "validation_ece": 0.01, + "gate_attrs_present": False, + "gate_all_pass": None, + }, + ] + + summary = _summarize_replayed_trials(rows, prioritize_gate_pass=True) + + assert summary["trial_number"].tolist() == [2, 3, 1] + assert summary["gate_tier"].tolist() == [0, 1, 2] + assert _gate_tier(True, prioritize_gate_pass=True) == 0 + assert _gate_tier(None, prioritize_gate_pass=True) == 1 + assert _gate_tier(False, prioritize_gate_pass=True) == 2 + assert _gate_tier(False, prioritize_gate_pass=False) == 1 + assert _replay_selection_policy(True)["rank_order"][0] == "gate_tier(pass->unknown->fail)" + + def test_load_training_splits_uses_config_paths_and_normalizes( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -342,3 +388,102 @@ def test_prepare_training_inputs_builds_model_ready_frames() -> None: assert x_test_cb["grade"].tolist() == ["B", "UNKNOWN"] assert x_test_lr.isna().sum().sum() == 0 assert lr_fill["score"] == x_train_fit_lr["score"].median() + + +def test_evaluate_walk_forward_stage_reports_disabled_without_training( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail_if_called(*args, **kwargs): + raise AssertionError("walk-forward evaluator should not run when disabled") + + monkeypatch.setattr(pd_train, "_evaluate_walk_forward_auc", fail_if_called) + + report = pd_train._evaluate_walk_forward_stage( + enabled=False, + walk_cfg={"n_windows": 5}, + train=pd_train.pd.DataFrame(), + catboost_features=[], + categorical_features=[], + model_params={}, + ) + + assert report == { + "enabled": False, + "reason": "disabled_in_config", + "n_windows_requested": 5, + "n_windows_used": 0, + "folds": [], + } + + +def test_evaluate_walk_forward_stage_normalizes_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + def fake_evaluate(train_df: pd_train.pd.DataFrame, **kwargs): + captured["train_rows"] = len(train_df) + captured.update(kwargs) + return {"enabled": True, "n_windows_used": 1} + + monkeypatch.setattr(pd_train, "_evaluate_walk_forward_auc", fake_evaluate) + + report = pd_train._evaluate_walk_forward_stage( + enabled=True, + walk_cfg={ + "n_windows": "4", + "min_train_rows": "12", + "window_rows": "6", + "date_col": "issue_d", + "max_rows": "0", + }, + train=pd_train.pd.DataFrame({"x": [1, 2]}), + catboost_features=["x"], + categorical_features=[], + model_params={"depth": 4}, + ) + + assert report == {"enabled": True, "n_windows_used": 1} + assert captured["features"] == ["x"] + assert captured["target"] == pd_train.TARGET + assert captured["params"] == {"depth": 4} + assert captured["n_windows"] == 4 + assert captured["min_train_rows"] == 12 + assert captured["window_rows"] == 6 + assert captured["max_rows"] is None + + +def test_export_shap_feature_importance_writes_summary( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_catboost = types.ModuleType("catboost") + + class FakePool: + def __init__(self, frame, cat_features): + self.frame = frame + self.cat_features = cat_features + + fake_catboost.Pool = FakePool + monkeypatch.setitem(sys.modules, "catboost", fake_catboost) + + class FakeModel: + def get_feature_importance(self, *, type: str, data: FakePool): + assert type == "ShapValues" + assert data.cat_features == ["grade"] + return np.array([[0.1, 0.4, 0.5], [0.3, -0.2, 0.5]]) + + shap_dir = tmp_path / "shap" + result = pd_train._export_shap_feature_importance( + cb_tuned_model=FakeModel(), + X_test_cb=pd_train.pd.DataFrame({"score": [1.0, 2.0], "grade": ["A", "B"]}), + categorical_features=["grade"], + catboost_features=["score", "grade"], + shap_dir=shap_dir, + ) + + summary = json.loads((shap_dir / "shap_feature_importance.json").read_text(encoding="utf-8")) + assert result == {"exported": True, "n_features": 2, "path": str(shap_dir)} + assert (shap_dir / "shap_values_test.npz").exists() + assert summary["expected_value"] == 0.5 + assert summary["top_features"][0]["feature"] == "grade" diff --git a/tests/test_scripts/test_validate_conformal_policy.py b/tests/test_scripts/test_validate_conformal_policy.py index e5ef542..2078000 100644 --- a/tests/test_scripts/test_validate_conformal_policy.py +++ b/tests/test_scripts/test_validate_conformal_policy.py @@ -65,6 +65,52 @@ def _fake_legacy_score(*_args, **_kwargs): assert score == pytest.approx(1.20) +def test_interval_arrays_filters_invalid_interval_rows() -> None: + intervals = pd.DataFrame( + { + "y_true": [0.1, None, 0.3, 0.4], + "pd_low_90": [0.0, 0.1, np.nan, 0.2], + "pd_high_90": [0.2, 0.3, 0.4, 0.6], + } + ) + + y_true, lower, upper = policy_mod._interval_arrays( + intervals, + lower_col="pd_low_90", + upper_col="pd_high_90", + ) + + assert list(y_true) == [0.1, 0.4] + assert list(lower) == [0.0, 0.2] + assert list(upper) == [0.2, 0.6] + + +def test_winkler_90_check_allows_documented_compensated_band() -> None: + policy = { + "target_coverage_90_min": 0.90, + "min_group_coverage_90_min": 0.88, + "max_avg_width_90": 0.80, + "max_critical_alerts": 0, + "max_winkler_90": 1.00, + "enable_compensated_winkler_90": True, + "compensated_winkler_90_max": 1.20, + } + metrics = { + "winkler_90": 1.10, + "coverage_90": 0.91, + "min_group_coverage_90": 0.89, + "avg_width_90": 0.70, + "critical_alerts": 0.0, + } + + check = policy_mod._winkler_90_check(policy, metrics) + + assert check["passed"] is True + assert check["raw_passed"] is False + assert check["compensated_passed"] is True + assert check["policy_mode"] == "compensated_band" + + def test_validate_conformal_policy_includes_material_gate_checks(tmp_path) -> None: data_dir = tmp_path / "data" / "processed" model_dir = tmp_path / "models" diff --git a/tests/test_supplement_table_sync.py b/tests/test_supplement_table_sync.py index d2fa0de..a425dcd 100644 --- a/tests/test_supplement_table_sync.py +++ b/tests/test_supplement_table_sync.py @@ -1,13 +1,9 @@ -"""Drift guard for hand-authored multidataset tables in the paper surfaces. +"""Drift guards for the diagnostic multidataset evidence. -The online supplement, the IJDS body, and book chapter 30 embed the external -replication tables (A25, A34) as hand-written Markdown rather than reading the -CSV at render time. That is convenient but can silently drift if a CSV is -regenerated without updating the prose. These tests assert the headline numbers -shown in those surfaces still match the source CSVs. - -Only a few anchor values are checked -- enough to catch a stale copy/paste, -without re-implementing a full Markdown table parser. +Chapter 30 retains the detailed A25/A34 values. The submitted body and +supplement deliberately keep only the scope boundary: these older replications +are static transfer evidence, not active-policy certificates. Tests preserve +both contracts without forcing retired detail back into the IJDS narrative. """ from __future__ import annotations @@ -39,27 +35,37 @@ def _text(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_a25_robust_objectives_match_surfaces() -> None: - """A25 robust LP objectives (as `$N`) appear in supplement, paper, and book.""" +def test_a25_robust_objectives_match_book_and_diagnostic_status() -> None: + """A25 values remain in the book while IJDS surfaces retain their boundary.""" rows = _read_rows("crpto_tableA25_external_replication_gate.csv") - surfaces = {p: _text(p) for p in (SUPPLEMENT, PAPER, BOOK_CH30)} + book = _text(BOOK_CH30) missing: list[str] = [] for row in rows: dollars = f"${round(float(row['robust_objective'])):,}" - for path, body in surfaces.items(): - if dollars not in body: - missing.append(f"{row['dataset']} {dollars} missing in {path.name}") + if dollars not in book: + missing.append(f"{row['dataset']} {dollars} missing in {BOOK_CH30.name}") assert not missing, "A25 robust objective drift:\n" + "\n".join(missing) + supplement = _text(SUPPLEMENT) + paper = _text(PAPER) + assert "A25--A34" in supplement + assert "Static transfer evidence; not active Lending Club certificates." in supplement + assert "Prosper" in paper and "Freddie/Mendeley" in paper + assert "retained as diagnostics or external context" in paper + -def test_a34_price_of_robustness_match_surfaces() -> None: - """A34 signed price-of-robustness (as `+X.XX%`) appears in supplement and book.""" +def test_a34_price_of_robustness_matches_historical_book_surface() -> None: + """A34 signed price-of-robustness values remain traceable in chapter 30.""" rows = _read_rows("crpto_tableA34_price_of_robustness_cross_dataset.csv") - surfaces = {p: _text(p) for p in (SUPPLEMENT, BOOK_CH30)} + book = _text(BOOK_CH30) missing: list[str] = [] for row in rows: pct = f"+{float(row['price_of_robustness_pct']) * 100:.2f}%" - for path, body in surfaces.items(): - if pct not in body: - missing.append(f"{row['application']} {pct} missing in {path.name}") + if pct not in book: + missing.append(f"{row['application']} {pct} missing in {BOOK_CH30.name}") assert not missing, "A34 price-of-robustness drift:\n" + "\n".join(missing) + + supplement = _text(SUPPLEMENT) + assert "older frozen replication contracts" in supplement + assert "cannot be quoted as direct" in supplement + assert "replications of the active midpoint policy" in supplement diff --git a/tests/test_utils/test_script_helpers.py b/tests/test_utils/test_script_helpers.py index 0ab892b..201d56d 100644 --- a/tests/test_utils/test_script_helpers.py +++ b/tests/test_utils/test_script_helpers.py @@ -17,6 +17,7 @@ parse_percent_series, policy_matches, resolve_interval_columns, + resolve_repo_artifact_path, try_load_json, write_json, write_table, @@ -77,6 +78,23 @@ def test_artifact_path_with_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) assert artifact_path("models/x.json") == tmp_path / "models/x.json" +def test_resolve_repo_artifact_path_maps_wsl_manifest_path(tmp_path: Path) -> None: + root = tmp_path / "Paper_CRPTO" + expected = root / "data" / "processed" / "intervals.parquet" + + resolved = resolve_repo_artifact_path( + "/mnt/c/Users/carlos/Documents/Paper_CRPTO/data/processed/intervals.parquet", + root=root, + ) + + assert resolved == expected + + +def test_resolve_repo_artifact_path_anchors_relative_path(tmp_path: Path) -> None: + root = tmp_path / "repo" + assert resolve_repo_artifact_path("models/a.json", root=root) == root / "models" / "a.json" + + def test_first_existing_prefers_existing(tmp_path: Path) -> None: existing = tmp_path / "exact.parquet" existing.touch()